001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs;
019    
020    import java.io.Closeable;
021    import java.io.FileNotFoundException;
022    import java.io.IOException;
023    import java.lang.ref.WeakReference;
024    import java.net.URI;
025    import java.net.URISyntaxException;
026    import java.security.PrivilegedExceptionAction;
027    import java.util.ArrayList;
028    import java.util.Arrays;
029    import java.util.Collections;
030    import java.util.EnumSet;
031    import java.util.HashMap;
032    import java.util.HashSet;
033    import java.util.IdentityHashMap;
034    import java.util.Iterator;
035    import java.util.LinkedList;
036    import java.util.List;
037    import java.util.Map;
038    import java.util.NoSuchElementException;
039    import java.util.ServiceLoader;
040    import java.util.Set;
041    import java.util.Stack;
042    import java.util.TreeSet;
043    import java.util.concurrent.atomic.AtomicInteger;
044    import java.util.concurrent.atomic.AtomicLong;
045    
046    import org.apache.commons.logging.Log;
047    import org.apache.commons.logging.LogFactory;
048    import org.apache.hadoop.classification.InterfaceAudience;
049    import org.apache.hadoop.classification.InterfaceStability;
050    import org.apache.hadoop.conf.Configuration;
051    import org.apache.hadoop.conf.Configured;
052    import org.apache.hadoop.fs.Options.ChecksumOpt;
053    import org.apache.hadoop.fs.Options.Rename;
054    import org.apache.hadoop.fs.permission.FsPermission;
055    import org.apache.hadoop.io.MultipleIOException;
056    import org.apache.hadoop.io.Text;
057    import org.apache.hadoop.net.NetUtils;
058    import org.apache.hadoop.security.Credentials;
059    import org.apache.hadoop.security.SecurityUtil;
060    import org.apache.hadoop.security.UserGroupInformation;
061    import org.apache.hadoop.security.token.Token;
062    import org.apache.hadoop.util.DataChecksum;
063    import org.apache.hadoop.util.Progressable;
064    import org.apache.hadoop.util.ReflectionUtils;
065    import org.apache.hadoop.util.ShutdownHookManager;
066    
067    import com.google.common.annotations.VisibleForTesting;
068    
069    /****************************************************************
070     * An abstract base class for a fairly generic filesystem.  It
071     * may be implemented as a distributed filesystem, or as a "local"
072     * one that reflects the locally-connected disk.  The local version
073     * exists for small Hadoop instances and for testing.
074     *
075     * <p>
076     *
077     * All user code that may potentially use the Hadoop Distributed
078     * File System should be written to use a FileSystem object.  The
079     * Hadoop DFS is a multi-machine system that appears as a single
080     * disk.  It's useful because of its fault tolerance and potentially
081     * very large capacity.
082     * 
083     * <p>
084     * The local implementation is {@link LocalFileSystem} and distributed
085     * implementation is DistributedFileSystem.
086     *****************************************************************/
087    @InterfaceAudience.Public
088    @InterfaceStability.Stable
089    public abstract class FileSystem extends Configured implements Closeable {
090      public static final String FS_DEFAULT_NAME_KEY = 
091                       CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
092      public static final String DEFAULT_FS = 
093                       CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
094    
095      public static final Log LOG = LogFactory.getLog(FileSystem.class);
096    
097      /**
098       * Priority of the FileSystem shutdown hook.
099       */
100      public static final int SHUTDOWN_HOOK_PRIORITY = 10;
101    
102      /** FileSystem cache */
103      static final Cache CACHE = new Cache();
104    
105      /** The key this instance is stored under in the cache. */
106      private Cache.Key key;
107    
108      /** Recording statistics per a FileSystem class */
109      private static final Map<Class<? extends FileSystem>, Statistics> 
110        statisticsTable =
111          new IdentityHashMap<Class<? extends FileSystem>, Statistics>();
112      
113      /**
114       * The statistics for this file system.
115       */
116      protected Statistics statistics;
117    
118      /**
119       * A cache of files that should be deleted when filsystem is closed
120       * or the JVM is exited.
121       */
122      private Set<Path> deleteOnExit = new TreeSet<Path>();
123      
124      /**
125       * This method adds a file system for testing so that we can find it later. It
126       * is only for testing.
127       * @param uri the uri to store it under
128       * @param conf the configuration to store it under
129       * @param fs the file system to store
130       * @throws IOException
131       */
132      static void addFileSystemForTesting(URI uri, Configuration conf,
133          FileSystem fs) throws IOException {
134        CACHE.map.put(new Cache.Key(uri, conf), fs);
135      }
136    
137      /**
138       * Get a filesystem instance based on the uri, the passed
139       * configuration and the user
140       * @param uri of the filesystem
141       * @param conf the configuration to use
142       * @param user to perform the get as
143       * @return the filesystem instance
144       * @throws IOException
145       * @throws InterruptedException
146       */
147      public static FileSystem get(final URI uri, final Configuration conf,
148            final String user) throws IOException, InterruptedException {
149        String ticketCachePath =
150          conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
151        UserGroupInformation ugi =
152            UserGroupInformation.getBestUGI(ticketCachePath, user);
153        return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
154          public FileSystem run() throws IOException {
155            return get(uri, conf);
156          }
157        });
158      }
159    
160      /**
161       * Returns the configured filesystem implementation.
162       * @param conf the configuration to use
163       */
164      public static FileSystem get(Configuration conf) throws IOException {
165        return get(getDefaultUri(conf), conf);
166      }
167      
168      /** Get the default filesystem URI from a configuration.
169       * @param conf the configuration to use
170       * @return the uri of the default filesystem
171       */
172      public static URI getDefaultUri(Configuration conf) {
173        return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
174      }
175    
176      /** Set the default filesystem URI in a configuration.
177       * @param conf the configuration to alter
178       * @param uri the new default filesystem uri
179       */
180      public static void setDefaultUri(Configuration conf, URI uri) {
181        conf.set(FS_DEFAULT_NAME_KEY, uri.toString());
182      }
183    
184      /** Set the default filesystem URI in a configuration.
185       * @param conf the configuration to alter
186       * @param uri the new default filesystem uri
187       */
188      public static void setDefaultUri(Configuration conf, String uri) {
189        setDefaultUri(conf, URI.create(fixName(uri)));
190      }
191    
192      /** Called after a new FileSystem instance is constructed.
193       * @param name a uri whose authority section names the host, port, etc.
194       *   for this FileSystem
195       * @param conf the configuration
196       */
197      public void initialize(URI name, Configuration conf) throws IOException {
198        statistics = getStatistics(name.getScheme(), getClass());    
199      }
200    
201      /**
202       * Return the protocol scheme for the FileSystem.
203       * <p/>
204       * This implementation throws an <code>UnsupportedOperationException</code>.
205       *
206       * @return the protocol scheme for the FileSystem.
207       */
208      public String getScheme() {
209        throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation");
210      }
211    
212      /** Returns a URI whose scheme and authority identify this FileSystem.*/
213      public abstract URI getUri();
214      
215      /**
216       * Return a canonicalized form of this FileSystem's URI.
217       * 
218       * The default implementation simply calls {@link #canonicalizeUri(URI)}
219       * on the filesystem's own URI, so subclasses typically only need to
220       * implement that method.
221       *
222       * @see #canonicalizeUri(URI)
223       */
224      protected URI getCanonicalUri() {
225        return canonicalizeUri(getUri());
226      }
227      
228      /**
229       * Canonicalize the given URI.
230       * 
231       * This is filesystem-dependent, but may for example consist of
232       * canonicalizing the hostname using DNS and adding the default
233       * port if not specified.
234       * 
235       * The default implementation simply fills in the default port if
236       * not specified and if the filesystem has a default port.
237       *
238       * @return URI
239       * @see NetUtils#getCanonicalUri(URI, int)
240       */
241      protected URI canonicalizeUri(URI uri) {
242        if (uri.getPort() == -1 && getDefaultPort() > 0) {
243          // reconstruct the uri with the default port set
244          try {
245            uri = new URI(uri.getScheme(), uri.getUserInfo(),
246                uri.getHost(), getDefaultPort(),
247                uri.getPath(), uri.getQuery(), uri.getFragment());
248          } catch (URISyntaxException e) {
249            // Should never happen!
250            throw new AssertionError("Valid URI became unparseable: " +
251                uri);
252          }
253        }
254        
255        return uri;
256      }
257      
258      /**
259       * Get the default port for this file system.
260       * @return the default port or 0 if there isn't one
261       */
262      protected int getDefaultPort() {
263        return 0;
264      }
265    
266      /**
267       * Get a canonical service name for this file system.  The token cache is
268       * the only user of the canonical service name, and uses it to lookup this
269       * filesystem's service tokens.
270       * If file system provides a token of its own then it must have a canonical
271       * name, otherwise canonical name can be null.
272       * 
273       * Default Impl: If the file system has child file systems 
274       * (such as an embedded file system) then it is assumed that the fs has no
275       * tokens of its own and hence returns a null name; otherwise a service
276       * name is built using Uri and port.
277       * 
278       * @return a service string that uniquely identifies this file system, null
279       *         if the filesystem does not implement tokens
280       * @see SecurityUtil#buildDTServiceName(URI, int) 
281       */
282      @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
283      public String getCanonicalServiceName() {
284        return (getChildFileSystems() == null)
285          ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort())
286          : null;
287      }
288    
289      /** @deprecated call #getUri() instead.*/
290      @Deprecated
291      public String getName() { return getUri().toString(); }
292    
293      /** @deprecated call #get(URI,Configuration) instead. */
294      @Deprecated
295      public static FileSystem getNamed(String name, Configuration conf)
296        throws IOException {
297        return get(URI.create(fixName(name)), conf);
298      }
299      
300      /** Update old-format filesystem names, for back-compatibility.  This should
301       * eventually be replaced with a checkName() method that throws an exception
302       * for old-format names. */ 
303      private static String fixName(String name) {
304        // convert old-format name to new-format name
305        if (name.equals("local")) {         // "local" is now "file:///".
306          LOG.warn("\"local\" is a deprecated filesystem name."
307                   +" Use \"file:///\" instead.");
308          name = "file:///";
309        } else if (name.indexOf('/')==-1) {   // unqualified is "hdfs://"
310          LOG.warn("\""+name+"\" is a deprecated filesystem name."
311                   +" Use \"hdfs://"+name+"/\" instead.");
312          name = "hdfs://"+name;
313        }
314        return name;
315      }
316    
317      /**
318       * Get the local file system.
319       * @param conf the configuration to configure the file system with
320       * @return a LocalFileSystem
321       */
322      public static LocalFileSystem getLocal(Configuration conf)
323        throws IOException {
324        return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
325      }
326    
327      /** Returns the FileSystem for this URI's scheme and authority.  The scheme
328       * of the URI determines a configuration property name,
329       * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
330       * The entire URI is passed to the FileSystem instance's initialize method.
331       */
332      public static FileSystem get(URI uri, Configuration conf) throws IOException {
333        String scheme = uri.getScheme();
334        String authority = uri.getAuthority();
335    
336        if (scheme == null && authority == null) {     // use default FS
337          return get(conf);
338        }
339    
340        if (scheme != null && authority == null) {     // no authority
341          URI defaultUri = getDefaultUri(conf);
342          if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
343              && defaultUri.getAuthority() != null) {  // & default has authority
344            return get(defaultUri, conf);              // return default
345          }
346        }
347        
348        String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
349        if (conf.getBoolean(disableCacheName, false)) {
350          return createFileSystem(uri, conf);
351        }
352    
353        return CACHE.get(uri, conf);
354      }
355    
356      /**
357       * Returns the FileSystem for this URI's scheme and authority and the 
358       * passed user. Internally invokes {@link #newInstance(URI, Configuration)}
359       * @param uri of the filesystem
360       * @param conf the configuration to use
361       * @param user to perform the get as
362       * @return filesystem instance
363       * @throws IOException
364       * @throws InterruptedException
365       */
366      public static FileSystem newInstance(final URI uri, final Configuration conf,
367          final String user) throws IOException, InterruptedException {
368        String ticketCachePath =
369          conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH);
370        UserGroupInformation ugi =
371            UserGroupInformation.getBestUGI(ticketCachePath, user);
372        return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
373          public FileSystem run() throws IOException {
374            return newInstance(uri,conf); 
375          }
376        });
377      }
378      /** Returns the FileSystem for this URI's scheme and authority.  The scheme
379       * of the URI determines a configuration property name,
380       * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
381       * The entire URI is passed to the FileSystem instance's initialize method.
382       * This always returns a new FileSystem object.
383       */
384      public static FileSystem newInstance(URI uri, Configuration conf) throws IOException {
385        String scheme = uri.getScheme();
386        String authority = uri.getAuthority();
387    
388        if (scheme == null) {                       // no scheme: use default FS
389          return newInstance(conf);
390        }
391    
392        if (authority == null) {                       // no authority
393          URI defaultUri = getDefaultUri(conf);
394          if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
395              && defaultUri.getAuthority() != null) {  // & default has authority
396            return newInstance(defaultUri, conf);              // return default
397          }
398        }
399        return CACHE.getUnique(uri, conf);
400      }
401    
402      /** Returns a unique configured filesystem implementation.
403       * This always returns a new FileSystem object.
404       * @param conf the configuration to use
405       */
406      public static FileSystem newInstance(Configuration conf) throws IOException {
407        return newInstance(getDefaultUri(conf), conf);
408      }
409    
410      /**
411       * Get a unique local file system object
412       * @param conf the configuration to configure the file system with
413       * @return a LocalFileSystem
414       * This always returns a new FileSystem object.
415       */
416      public static LocalFileSystem newInstanceLocal(Configuration conf)
417        throws IOException {
418        return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf);
419      }
420    
421      /**
422       * Close all cached filesystems. Be sure those filesystems are not
423       * used anymore.
424       * 
425       * @throws IOException
426       */
427      public static void closeAll() throws IOException {
428        CACHE.closeAll();
429      }
430    
431      /**
432       * Close all cached filesystems for a given UGI. Be sure those filesystems 
433       * are not used anymore.
434       * @param ugi user group info to close
435       * @throws IOException
436       */
437      public static void closeAllForUGI(UserGroupInformation ugi) 
438      throws IOException {
439        CACHE.closeAll(ugi);
440      }
441    
442      /** 
443       * Make sure that a path specifies a FileSystem.
444       * @param path to use
445       */
446      public Path makeQualified(Path path) {
447        checkPath(path);
448        return path.makeQualified(this.getUri(), this.getWorkingDirectory());
449      }
450        
451      /**
452       * Get a new delegation token for this file system.
453       * This is an internal method that should have been declared protected
454       * but wasn't historically.
455       * Callers should use {@link #addDelegationTokens(String, Credentials)}
456       * 
457       * @param renewer the account name that is allowed to renew the token.
458       * @return a new delegation token
459       * @throws IOException
460       */
461      @InterfaceAudience.Private()
462      public Token<?> getDelegationToken(String renewer) throws IOException {
463        return null;
464      }
465      
466      /**
467       * Obtain all delegation tokens used by this FileSystem that are not
468       * already present in the given Credentials.  Existing tokens will neither
469       * be verified as valid nor having the given renewer.  Missing tokens will
470       * be acquired and added to the given Credentials.
471       * 
472       * Default Impl: works for simple fs with its own token
473       * and also for an embedded fs whose tokens are those of its
474       * children file system (i.e. the embedded fs has not tokens of its
475       * own).
476       * 
477       * @param renewer the user allowed to renew the delegation tokens
478       * @param credentials cache in which to add new delegation tokens
479       * @return list of new delegation tokens
480       * @throws IOException
481       */
482      @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
483      public Token<?>[] addDelegationTokens(
484          final String renewer, Credentials credentials) throws IOException {
485        if (credentials == null) {
486          credentials = new Credentials();
487        }
488        final List<Token<?>> tokens = new ArrayList<Token<?>>();
489        collectDelegationTokens(renewer, credentials, tokens);
490        return tokens.toArray(new Token<?>[tokens.size()]);
491      }
492      
493      /**
494       * Recursively obtain the tokens for this FileSystem and all descended
495       * FileSystems as determined by getChildFileSystems().
496       * @param renewer the user allowed to renew the delegation tokens
497       * @param credentials cache in which to add the new delegation tokens
498       * @param tokens list in which to add acquired tokens
499       * @throws IOException
500       */
501      private void collectDelegationTokens(final String renewer,
502                                           final Credentials credentials,
503                                           final List<Token<?>> tokens)
504                                               throws IOException {
505        final String serviceName = getCanonicalServiceName();
506        // Collect token of the this filesystem and then of its embedded children
507        if (serviceName != null) { // fs has token, grab it
508          final Text service = new Text(serviceName);
509          Token<?> token = credentials.getToken(service);
510          if (token == null) {
511            token = getDelegationToken(renewer);
512            if (token != null) {
513              tokens.add(token);
514              credentials.addToken(service, token);
515            }
516          }
517        }
518        // Now collect the tokens from the children
519        final FileSystem[] children = getChildFileSystems();
520        if (children != null) {
521          for (final FileSystem fs : children) {
522            fs.collectDelegationTokens(renewer, credentials, tokens);
523          }
524        }
525      }
526    
527      /**
528       * Get all the immediate child FileSystems embedded in this FileSystem.
529       * It does not recurse and get grand children.  If a FileSystem
530       * has multiple child FileSystems, then it should return a unique list
531       * of those FileSystems.  Default is to return null to signify no children.
532       * 
533       * @return FileSystems used by this FileSystem
534       */
535      @InterfaceAudience.LimitedPrivate({ "HDFS" })
536      @VisibleForTesting
537      public FileSystem[] getChildFileSystems() {
538        return null;
539      }
540      
541      /** create a file with the provided permission
542       * The permission of the file is set to be the provided permission as in
543       * setPermission, not permission&~umask
544       * 
545       * It is implemented using two RPCs. It is understood that it is inefficient,
546       * but the implementation is thread-safe. The other option is to change the
547       * value of umask in configuration to be 0, but it is not thread-safe.
548       * 
549       * @param fs file system handle
550       * @param file the name of the file to be created
551       * @param permission the permission of the file
552       * @return an output stream
553       * @throws IOException
554       */
555      public static FSDataOutputStream create(FileSystem fs,
556          Path file, FsPermission permission) throws IOException {
557        // create the file with default permission
558        FSDataOutputStream out = fs.create(file);
559        // set its permission to the supplied one
560        fs.setPermission(file, permission);
561        return out;
562      }
563    
564      /** create a directory with the provided permission
565       * The permission of the directory is set to be the provided permission as in
566       * setPermission, not permission&~umask
567       * 
568       * @see #create(FileSystem, Path, FsPermission)
569       * 
570       * @param fs file system handle
571       * @param dir the name of the directory to be created
572       * @param permission the permission of the directory
573       * @return true if the directory creation succeeds; false otherwise
574       * @throws IOException
575       */
576      public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission)
577      throws IOException {
578        // create the directory using the default permission
579        boolean result = fs.mkdirs(dir);
580        // set its permission to be the supplied one
581        fs.setPermission(dir, permission);
582        return result;
583      }
584    
585      ///////////////////////////////////////////////////////////////
586      // FileSystem
587      ///////////////////////////////////////////////////////////////
588    
589      protected FileSystem() {
590        super(null);
591      }
592    
593      /** 
594       * Check that a Path belongs to this FileSystem.
595       * @param path to check
596       */
597      protected void checkPath(Path path) {
598        URI uri = path.toUri();
599        String thatScheme = uri.getScheme();
600        if (thatScheme == null)                // fs is relative
601          return;
602        URI thisUri = getCanonicalUri();
603        String thisScheme = thisUri.getScheme();
604        //authority and scheme are not case sensitive
605        if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match
606          String thisAuthority = thisUri.getAuthority();
607          String thatAuthority = uri.getAuthority();
608          if (thatAuthority == null &&                // path's authority is null
609              thisAuthority != null) {                // fs has an authority
610            URI defaultUri = getDefaultUri(getConf());
611            if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) {
612              uri = defaultUri; // schemes match, so use this uri instead
613            } else {
614              uri = null; // can't determine auth of the path
615            }
616          }
617          if (uri != null) {
618            // canonicalize uri before comparing with this fs
619            uri = canonicalizeUri(uri);
620            thatAuthority = uri.getAuthority();
621            if (thisAuthority == thatAuthority ||       // authorities match
622                (thisAuthority != null &&
623                 thisAuthority.equalsIgnoreCase(thatAuthority)))
624              return;
625          }
626        }
627        throw new IllegalArgumentException("Wrong FS: "+path+
628                                           ", expected: "+this.getUri());
629      }
630    
631      /**
632       * Return an array containing hostnames, offset and size of 
633       * portions of the given file.  For a nonexistent 
634       * file or regions, null will be returned.
635       *
636       * This call is most helpful with DFS, where it returns 
637       * hostnames of machines that contain the given file.
638       *
639       * The FileSystem will simply return an elt containing 'localhost'.
640       *
641       * @param file FilesStatus to get data from
642       * @param start offset into the given file
643       * @param len length for which to get locations for
644       */
645      public BlockLocation[] getFileBlockLocations(FileStatus file, 
646          long start, long len) throws IOException {
647        if (file == null) {
648          return null;
649        }
650    
651        if (start < 0 || len < 0) {
652          throw new IllegalArgumentException("Invalid start or len parameter");
653        }
654    
655        if (file.getLen() <= start) {
656          return new BlockLocation[0];
657    
658        }
659        String[] name = { "localhost:50010" };
660        String[] host = { "localhost" };
661        return new BlockLocation[] {
662          new BlockLocation(name, host, 0, file.getLen()) };
663      }
664     
665    
666      /**
667       * Return an array containing hostnames, offset and size of 
668       * portions of the given file.  For a nonexistent 
669       * file or regions, null will be returned.
670       *
671       * This call is most helpful with DFS, where it returns 
672       * hostnames of machines that contain the given file.
673       *
674       * The FileSystem will simply return an elt containing 'localhost'.
675       *
676       * @param p path is used to identify an FS since an FS could have
677       *          another FS that it could be delegating the call to
678       * @param start offset into the given file
679       * @param len length for which to get locations for
680       */
681      public BlockLocation[] getFileBlockLocations(Path p, 
682          long start, long len) throws IOException {
683        if (p == null) {
684          throw new NullPointerException();
685        }
686        FileStatus file = getFileStatus(p);
687        return getFileBlockLocations(file, start, len);
688      }
689      
690      /**
691       * Return a set of server default configuration values
692       * @return server default configuration values
693       * @throws IOException
694       * @deprecated use {@link #getServerDefaults(Path)} instead
695       */
696      @Deprecated
697      public FsServerDefaults getServerDefaults() throws IOException {
698        Configuration conf = getConf();
699        // CRC32 is chosen as default as it is available in all 
700        // releases that support checksum.
701        // The client trash configuration is ignored.
702        return new FsServerDefaults(getDefaultBlockSize(), 
703            conf.getInt("io.bytes.per.checksum", 512), 
704            64 * 1024, 
705            getDefaultReplication(),
706            conf.getInt("io.file.buffer.size", 4096),
707            false,
708            CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT,
709            DataChecksum.CHECKSUM_CRC32);
710      }
711    
712      /**
713       * Return a set of server default configuration values
714       * @param p path is used to identify an FS since an FS could have
715       *          another FS that it could be delegating the call to
716       * @return server default configuration values
717       * @throws IOException
718       */
719      public FsServerDefaults getServerDefaults(Path p) throws IOException {
720        return getServerDefaults();
721      }
722    
723      /**
724       * Return the fully-qualified path of path f resolving the path
725       * through any symlinks or mount point
726       * @param p path to be resolved
727       * @return fully qualified path 
728       * @throws FileNotFoundException
729       */
730       public Path resolvePath(final Path p) throws IOException {
731         checkPath(p);
732         return getFileStatus(p).getPath();
733       }
734    
735      /**
736       * Opens an FSDataInputStream at the indicated Path.
737       * @param f the file name to open
738       * @param bufferSize the size of the buffer to be used.
739       */
740      public abstract FSDataInputStream open(Path f, int bufferSize)
741        throws IOException;
742        
743      /**
744       * Opens an FSDataInputStream at the indicated Path.
745       * @param f the file to open
746       */
747      public FSDataInputStream open(Path f) throws IOException {
748        return open(f, getConf().getInt("io.file.buffer.size", 4096));
749      }
750    
751      /**
752       * Create an FSDataOutputStream at the indicated Path.
753       * Files are overwritten by default.
754       * @param f the file to create
755       */
756      public FSDataOutputStream create(Path f) throws IOException {
757        return create(f, true);
758      }
759    
760      /**
761       * Create an FSDataOutputStream at the indicated Path.
762       * @param f the file to create
763       * @param overwrite if a file with this name already exists, then if true,
764       *   the file will be overwritten, and if false an exception will be thrown.
765       */
766      public FSDataOutputStream create(Path f, boolean overwrite)
767          throws IOException {
768        return create(f, overwrite, 
769                      getConf().getInt("io.file.buffer.size", 4096),
770                      getDefaultReplication(f),
771                      getDefaultBlockSize(f));
772      }
773    
774      /**
775       * Create an FSDataOutputStream at the indicated Path with write-progress
776       * reporting.
777       * Files are overwritten by default.
778       * @param f the file to create
779       * @param progress to report progress
780       */
781      public FSDataOutputStream create(Path f, Progressable progress) 
782          throws IOException {
783        return create(f, true, 
784                      getConf().getInt("io.file.buffer.size", 4096),
785                      getDefaultReplication(f),
786                      getDefaultBlockSize(f), progress);
787      }
788    
789      /**
790       * Create an FSDataOutputStream at the indicated Path.
791       * Files are overwritten by default.
792       * @param f the file to create
793       * @param replication the replication factor
794       */
795      public FSDataOutputStream create(Path f, short replication)
796          throws IOException {
797        return create(f, true, 
798                      getConf().getInt("io.file.buffer.size", 4096),
799                      replication,
800                      getDefaultBlockSize(f));
801      }
802    
803      /**
804       * Create an FSDataOutputStream at the indicated Path with write-progress
805       * reporting.
806       * Files are overwritten by default.
807       * @param f the file to create
808       * @param replication the replication factor
809       * @param progress to report progress
810       */
811      public FSDataOutputStream create(Path f, short replication, 
812          Progressable progress) throws IOException {
813        return create(f, true, 
814                      getConf().getInt("io.file.buffer.size", 4096),
815                      replication,
816                      getDefaultBlockSize(f), progress);
817      }
818    
819        
820      /**
821       * Create an FSDataOutputStream at the indicated Path.
822       * @param f the file name to create
823       * @param overwrite if a file with this name already exists, then if true,
824       *   the file will be overwritten, and if false an error will be thrown.
825       * @param bufferSize the size of the buffer to be used.
826       */
827      public FSDataOutputStream create(Path f, 
828                                       boolean overwrite,
829                                       int bufferSize
830                                       ) throws IOException {
831        return create(f, overwrite, bufferSize, 
832                      getDefaultReplication(f),
833                      getDefaultBlockSize(f));
834      }
835        
836      /**
837       * Create an FSDataOutputStream at the indicated Path with write-progress
838       * reporting.
839       * @param f the path of the file to open
840       * @param overwrite if a file with this name already exists, then if true,
841       *   the file will be overwritten, and if false an error will be thrown.
842       * @param bufferSize the size of the buffer to be used.
843       */
844      public FSDataOutputStream create(Path f, 
845                                       boolean overwrite,
846                                       int bufferSize,
847                                       Progressable progress
848                                       ) throws IOException {
849        return create(f, overwrite, bufferSize, 
850                      getDefaultReplication(f),
851                      getDefaultBlockSize(f), progress);
852      }
853        
854        
855      /**
856       * Create an FSDataOutputStream at the indicated Path.
857       * @param f the file name to open
858       * @param overwrite if a file with this name already exists, then if true,
859       *   the file will be overwritten, and if false an error will be thrown.
860       * @param bufferSize the size of the buffer to be used.
861       * @param replication required block replication for the file. 
862       */
863      public FSDataOutputStream create(Path f, 
864                                       boolean overwrite,
865                                       int bufferSize,
866                                       short replication,
867                                       long blockSize
868                                       ) throws IOException {
869        return create(f, overwrite, bufferSize, replication, blockSize, null);
870      }
871    
872      /**
873       * Create an FSDataOutputStream at the indicated Path with write-progress
874       * reporting.
875       * @param f the file name to open
876       * @param overwrite if a file with this name already exists, then if true,
877       *   the file will be overwritten, and if false an error will be thrown.
878       * @param bufferSize the size of the buffer to be used.
879       * @param replication required block replication for the file. 
880       */
881      public FSDataOutputStream create(Path f,
882                                                boolean overwrite,
883                                                int bufferSize,
884                                                short replication,
885                                                long blockSize,
886                                                Progressable progress
887                                                ) throws IOException {
888        return this.create(f, FsPermission.getFileDefault().applyUMask(
889            FsPermission.getUMask(getConf())), overwrite, bufferSize,
890            replication, blockSize, progress);
891      }
892    
893      /**
894       * Create an FSDataOutputStream at the indicated Path with write-progress
895       * reporting.
896       * @param f the file name to open
897       * @param permission
898       * @param overwrite if a file with this name already exists, then if true,
899       *   the file will be overwritten, and if false an error will be thrown.
900       * @param bufferSize the size of the buffer to be used.
901       * @param replication required block replication for the file.
902       * @param blockSize
903       * @param progress
904       * @throws IOException
905       * @see #setPermission(Path, FsPermission)
906       */
907      public abstract FSDataOutputStream create(Path f,
908          FsPermission permission,
909          boolean overwrite,
910          int bufferSize,
911          short replication,
912          long blockSize,
913          Progressable progress) throws IOException;
914      
915      /**
916       * Create an FSDataOutputStream at the indicated Path with write-progress
917       * reporting.
918       * @param f the file name to open
919       * @param permission
920       * @param flags {@link CreateFlag}s to use for this stream.
921       * @param bufferSize the size of the buffer to be used.
922       * @param replication required block replication for the file.
923       * @param blockSize
924       * @param progress
925       * @throws IOException
926       * @see #setPermission(Path, FsPermission)
927       */
928      public FSDataOutputStream create(Path f,
929          FsPermission permission,
930          EnumSet<CreateFlag> flags,
931          int bufferSize,
932          short replication,
933          long blockSize,
934          Progressable progress) throws IOException {
935        return create(f, permission, flags, bufferSize, replication,
936            blockSize, progress, null);
937      }
938      
939      /**
940       * Create an FSDataOutputStream at the indicated Path with a custom
941       * checksum option
942       * @param f the file name to open
943       * @param permission
944       * @param flags {@link CreateFlag}s to use for this stream.
945       * @param bufferSize the size of the buffer to be used.
946       * @param replication required block replication for the file.
947       * @param blockSize
948       * @param progress
949       * @param checksumOpt checksum parameter. If null, the values
950       *        found in conf will be used.
951       * @throws IOException
952       * @see #setPermission(Path, FsPermission)
953       */
954      public FSDataOutputStream create(Path f,
955          FsPermission permission,
956          EnumSet<CreateFlag> flags,
957          int bufferSize,
958          short replication,
959          long blockSize,
960          Progressable progress,
961          ChecksumOpt checksumOpt) throws IOException {
962        // Checksum options are ignored by default. The file systems that
963        // implement checksum need to override this method. The full
964        // support is currently only available in DFS.
965        return create(f, permission, flags.contains(CreateFlag.OVERWRITE), 
966            bufferSize, replication, blockSize, progress);
967      }
968    
969      /*.
970       * This create has been added to support the FileContext that processes
971       * the permission
972       * with umask before calling this method.
973       * This a temporary method added to support the transition from FileSystem
974       * to FileContext for user applications.
975       */
976      @Deprecated
977      protected FSDataOutputStream primitiveCreate(Path f,
978         FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
979         short replication, long blockSize, Progressable progress,
980         ChecksumOpt checksumOpt) throws IOException {
981    
982        boolean pathExists = exists(f);
983        CreateFlag.validate(f, pathExists, flag);
984        
985        // Default impl  assumes that permissions do not matter and 
986        // nor does the bytesPerChecksum  hence
987        // calling the regular create is good enough.
988        // FSs that implement permissions should override this.
989    
990        if (pathExists && flag.contains(CreateFlag.APPEND)) {
991          return append(f, bufferSize, progress);
992        }
993        
994        return this.create(f, absolutePermission,
995            flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
996            blockSize, progress);
997      }
998      
999      /**
1000       * This version of the mkdirs method assumes that the permission is absolute.
1001       * It has been added to support the FileContext that processes the permission
1002       * with umask before calling this method.
1003       * This a temporary method added to support the transition from FileSystem
1004       * to FileContext for user applications.
1005       */
1006      @Deprecated
1007      protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
1008        throws IOException {
1009        // Default impl is to assume that permissions do not matter and hence
1010        // calling the regular mkdirs is good enough.
1011        // FSs that implement permissions should override this.
1012       return this.mkdirs(f, absolutePermission);
1013      }
1014    
1015    
1016      /**
1017       * This version of the mkdirs method assumes that the permission is absolute.
1018       * It has been added to support the FileContext that processes the permission
1019       * with umask before calling this method.
1020       * This a temporary method added to support the transition from FileSystem
1021       * to FileContext for user applications.
1022       */
1023      @Deprecated
1024      protected void primitiveMkdir(Path f, FsPermission absolutePermission, 
1025                        boolean createParent)
1026        throws IOException {
1027        
1028        if (!createParent) { // parent must exist.
1029          // since the this.mkdirs makes parent dirs automatically
1030          // we must throw exception if parent does not exist.
1031          final FileStatus stat = getFileStatus(f.getParent());
1032          if (stat == null) {
1033            throw new FileNotFoundException("Missing parent:" + f);
1034          }
1035          if (!stat.isDirectory()) {
1036            throw new ParentNotDirectoryException("parent is not a dir");
1037          }
1038          // parent does exist - go ahead with mkdir of leaf
1039        }
1040        // Default impl is to assume that permissions do not matter and hence
1041        // calling the regular mkdirs is good enough.
1042        // FSs that implement permissions should override this.
1043        if (!this.mkdirs(f, absolutePermission)) {
1044          throw new IOException("mkdir of "+ f + " failed");
1045        }
1046      }
1047    
1048      /**
1049       * Opens an FSDataOutputStream at the indicated Path with write-progress
1050       * reporting. Same as create(), except fails if parent directory doesn't
1051       * already exist.
1052       * @param f the file name to open
1053       * @param overwrite if a file with this name already exists, then if true,
1054       * the file will be overwritten, and if false an error will be thrown.
1055       * @param bufferSize the size of the buffer to be used.
1056       * @param replication required block replication for the file.
1057       * @param blockSize
1058       * @param progress
1059       * @throws IOException
1060       * @see #setPermission(Path, FsPermission)
1061       * @deprecated API only for 0.20-append
1062       */
1063      @Deprecated
1064      public FSDataOutputStream createNonRecursive(Path f,
1065          boolean overwrite,
1066          int bufferSize, short replication, long blockSize,
1067          Progressable progress) throws IOException {
1068        return this.createNonRecursive(f, FsPermission.getFileDefault(),
1069            overwrite, bufferSize, replication, blockSize, progress);
1070      }
1071    
1072      /**
1073       * Opens an FSDataOutputStream at the indicated Path with write-progress
1074       * reporting. Same as create(), except fails if parent directory doesn't
1075       * already exist.
1076       * @param f the file name to open
1077       * @param permission
1078       * @param overwrite if a file with this name already exists, then if true,
1079       * the file will be overwritten, and if false an error will be thrown.
1080       * @param bufferSize the size of the buffer to be used.
1081       * @param replication required block replication for the file.
1082       * @param blockSize
1083       * @param progress
1084       * @throws IOException
1085       * @see #setPermission(Path, FsPermission)
1086       * @deprecated API only for 0.20-append
1087       */
1088       @Deprecated
1089       public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
1090           boolean overwrite, int bufferSize, short replication, long blockSize,
1091           Progressable progress) throws IOException {
1092         return createNonRecursive(f, permission,
1093             overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
1094                 : EnumSet.of(CreateFlag.CREATE), bufferSize,
1095                 replication, blockSize, progress);
1096       }
1097    
1098       /**
1099        * Opens an FSDataOutputStream at the indicated Path with write-progress
1100        * reporting. Same as create(), except fails if parent directory doesn't
1101        * already exist.
1102        * @param f the file name to open
1103        * @param permission
1104        * @param flags {@link CreateFlag}s to use for this stream.
1105        * @param bufferSize the size of the buffer to be used.
1106        * @param replication required block replication for the file.
1107        * @param blockSize
1108        * @param progress
1109        * @throws IOException
1110        * @see #setPermission(Path, FsPermission)
1111        * @deprecated API only for 0.20-append
1112        */
1113        @Deprecated
1114        public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
1115            EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
1116            Progressable progress) throws IOException {
1117          throw new IOException("createNonRecursive unsupported for this filesystem "
1118              + this.getClass());
1119        }
1120    
1121      /**
1122       * Creates the given Path as a brand-new zero-length file.  If
1123       * create fails, or if it already existed, return false.
1124       *
1125       * @param f path to use for create
1126       */
1127      public boolean createNewFile(Path f) throws IOException {
1128        if (exists(f)) {
1129          return false;
1130        } else {
1131          create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
1132          return true;
1133        }
1134      }
1135    
1136      /**
1137       * Append to an existing file (optional operation).
1138       * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
1139       * @param f the existing file to be appended.
1140       * @throws IOException
1141       */
1142      public FSDataOutputStream append(Path f) throws IOException {
1143        return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
1144      }
1145      /**
1146       * Append to an existing file (optional operation).
1147       * Same as append(f, bufferSize, null).
1148       * @param f the existing file to be appended.
1149       * @param bufferSize the size of the buffer to be used.
1150       * @throws IOException
1151       */
1152      public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
1153        return append(f, bufferSize, null);
1154      }
1155    
1156      /**
1157       * Append to an existing file (optional operation).
1158       * @param f the existing file to be appended.
1159       * @param bufferSize the size of the buffer to be used.
1160       * @param progress for reporting progress if it is not null.
1161       * @throws IOException
1162       */
1163      public abstract FSDataOutputStream append(Path f, int bufferSize,
1164          Progressable progress) throws IOException;
1165    
1166      /**
1167       * Concat existing files together.
1168       * @param trg the path to the target destination.
1169       * @param psrcs the paths to the sources to use for the concatenation.
1170       * @throws IOException
1171       */
1172      public void concat(final Path trg, final Path [] psrcs) throws IOException {
1173        throw new UnsupportedOperationException("Not implemented by the " + 
1174            getClass().getSimpleName() + " FileSystem implementation");
1175      }
1176    
1177     /**
1178       * Get replication.
1179       * 
1180       * @deprecated Use getFileStatus() instead
1181       * @param src file name
1182       * @return file replication
1183       * @throws IOException
1184       */ 
1185      @Deprecated
1186      public short getReplication(Path src) throws IOException {
1187        return getFileStatus(src).getReplication();
1188      }
1189    
1190      /**
1191       * Set replication for an existing file.
1192       * 
1193       * @param src file name
1194       * @param replication new replication
1195       * @throws IOException
1196       * @return true if successful;
1197       *         false if file does not exist or is a directory
1198       */
1199      public boolean setReplication(Path src, short replication)
1200        throws IOException {
1201        return true;
1202      }
1203    
1204      /**
1205       * Renames Path src to Path dst.  Can take place on local fs
1206       * or remote DFS.
1207       * @param src path to be renamed
1208       * @param dst new path after rename
1209       * @throws IOException on failure
1210       * @return true if rename is successful
1211       */
1212      public abstract boolean rename(Path src, Path dst) throws IOException;
1213    
1214      /**
1215       * Renames Path src to Path dst
1216       * <ul>
1217       * <li
1218       * <li>Fails if src is a file and dst is a directory.
1219       * <li>Fails if src is a directory and dst is a file.
1220       * <li>Fails if the parent of dst does not exist or is a file.
1221       * </ul>
1222       * <p>
1223       * If OVERWRITE option is not passed as an argument, rename fails
1224       * if the dst already exists.
1225       * <p>
1226       * If OVERWRITE option is passed as an argument, rename overwrites
1227       * the dst if it is a file or an empty directory. Rename fails if dst is
1228       * a non-empty directory.
1229       * <p>
1230       * Note that atomicity of rename is dependent on the file system
1231       * implementation. Please refer to the file system documentation for
1232       * details. This default implementation is non atomic.
1233       * <p>
1234       * This method is deprecated since it is a temporary method added to 
1235       * support the transition from FileSystem to FileContext for user 
1236       * applications.
1237       * 
1238       * @param src path to be renamed
1239       * @param dst new path after rename
1240       * @throws IOException on failure
1241       */
1242      @Deprecated
1243      protected void rename(final Path src, final Path dst,
1244          final Rename... options) throws IOException {
1245        // Default implementation
1246        final FileStatus srcStatus = getFileStatus(src);
1247        if (srcStatus == null) {
1248          throw new FileNotFoundException("rename source " + src + " not found.");
1249        }
1250    
1251        boolean overwrite = false;
1252        if (null != options) {
1253          for (Rename option : options) {
1254            if (option == Rename.OVERWRITE) {
1255              overwrite = true;
1256            }
1257          }
1258        }
1259    
1260        FileStatus dstStatus;
1261        try {
1262          dstStatus = getFileStatus(dst);
1263        } catch (IOException e) {
1264          dstStatus = null;
1265        }
1266        if (dstStatus != null) {
1267          if (srcStatus.isDirectory() != dstStatus.isDirectory()) {
1268            throw new IOException("Source " + src + " Destination " + dst
1269                + " both should be either file or directory");
1270          }
1271          if (!overwrite) {
1272            throw new FileAlreadyExistsException("rename destination " + dst
1273                + " already exists.");
1274          }
1275          // Delete the destination that is a file or an empty directory
1276          if (dstStatus.isDirectory()) {
1277            FileStatus[] list = listStatus(dst);
1278            if (list != null && list.length != 0) {
1279              throw new IOException(
1280                  "rename cannot overwrite non empty destination directory " + dst);
1281            }
1282          }
1283          delete(dst, false);
1284        } else {
1285          final Path parent = dst.getParent();
1286          final FileStatus parentStatus = getFileStatus(parent);
1287          if (parentStatus == null) {
1288            throw new FileNotFoundException("rename destination parent " + parent
1289                + " not found.");
1290          }
1291          if (!parentStatus.isDirectory()) {
1292            throw new ParentNotDirectoryException("rename destination parent " + parent
1293                + " is a file.");
1294          }
1295        }
1296        if (!rename(src, dst)) {
1297          throw new IOException("rename from " + src + " to " + dst + " failed.");
1298        }
1299      }
1300      
1301      /**
1302       * Delete a file 
1303       * @deprecated Use {@link #delete(Path, boolean)} instead.
1304       */
1305      @Deprecated
1306      public boolean delete(Path f) throws IOException {
1307        return delete(f, true);
1308      }
1309      
1310      /** Delete a file.
1311       *
1312       * @param f the path to delete.
1313       * @param recursive if path is a directory and set to 
1314       * true, the directory is deleted else throws an exception. In
1315       * case of a file the recursive can be set to either true or false. 
1316       * @return  true if delete is successful else false. 
1317       * @throws IOException
1318       */
1319      public abstract boolean delete(Path f, boolean recursive) throws IOException;
1320    
1321      /**
1322       * Mark a path to be deleted when FileSystem is closed.
1323       * When the JVM shuts down,
1324       * all FileSystem objects will be closed automatically.
1325       * Then,
1326       * the marked path will be deleted as a result of closing the FileSystem.
1327       *
1328       * The path has to exist in the file system.
1329       * 
1330       * @param f the path to delete.
1331       * @return  true if deleteOnExit is successful, otherwise false.
1332       * @throws IOException
1333       */
1334      public boolean deleteOnExit(Path f) throws IOException {
1335        if (!exists(f)) {
1336          return false;
1337        }
1338        synchronized (deleteOnExit) {
1339          deleteOnExit.add(f);
1340        }
1341        return true;
1342      }
1343      
1344      /**
1345       * Cancel the deletion of the path when the FileSystem is closed
1346       * @param f the path to cancel deletion
1347       */
1348      public boolean cancelDeleteOnExit(Path f) {
1349        synchronized (deleteOnExit) {
1350          return deleteOnExit.remove(f);
1351        }
1352      }
1353    
1354      /**
1355       * Delete all files that were marked as delete-on-exit. This recursively
1356       * deletes all files in the specified paths.
1357       */
1358      protected void processDeleteOnExit() {
1359        synchronized (deleteOnExit) {
1360          for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
1361            Path path = iter.next();
1362            try {
1363              if (exists(path)) {
1364                delete(path, true);
1365              }
1366            }
1367            catch (IOException e) {
1368              LOG.info("Ignoring failure to deleteOnExit for path " + path);
1369            }
1370            iter.remove();
1371          }
1372        }
1373      }
1374      
1375      /** Check if exists.
1376       * @param f source file
1377       */
1378      public boolean exists(Path f) throws IOException {
1379        try {
1380          return getFileStatus(f) != null;
1381        } catch (FileNotFoundException e) {
1382          return false;
1383        }
1384      }
1385    
1386      /** True iff the named path is a directory.
1387       * Note: Avoid using this method. Instead reuse the FileStatus 
1388       * returned by getFileStatus() or listStatus() methods.
1389       * @param f path to check
1390       */
1391      public boolean isDirectory(Path f) throws IOException {
1392        try {
1393          return getFileStatus(f).isDirectory();
1394        } catch (FileNotFoundException e) {
1395          return false;               // f does not exist
1396        }
1397      }
1398    
1399      /** True iff the named path is a regular file.
1400       * Note: Avoid using this method. Instead reuse the FileStatus 
1401       * returned by getFileStatus() or listStatus() methods.
1402       * @param f path to check
1403       */
1404      public boolean isFile(Path f) throws IOException {
1405        try {
1406          return getFileStatus(f).isFile();
1407        } catch (FileNotFoundException e) {
1408          return false;               // f does not exist
1409        }
1410      }
1411      
1412      /** The number of bytes in a file. */
1413      /** @deprecated Use getFileStatus() instead */
1414      @Deprecated
1415      public long getLength(Path f) throws IOException {
1416        return getFileStatus(f).getLen();
1417      }
1418        
1419      /** Return the {@link ContentSummary} of a given {@link Path}.
1420      * @param f path to use
1421      */
1422      public ContentSummary getContentSummary(Path f) throws IOException {
1423        FileStatus status = getFileStatus(f);
1424        if (status.isFile()) {
1425          // f is a file
1426          return new ContentSummary(status.getLen(), 1, 0);
1427        }
1428        // f is a directory
1429        long[] summary = {0, 0, 1};
1430        for(FileStatus s : listStatus(f)) {
1431          ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1432                                         new ContentSummary(s.getLen(), 1, 0);
1433          summary[0] += c.getLength();
1434          summary[1] += c.getFileCount();
1435          summary[2] += c.getDirectoryCount();
1436        }
1437        return new ContentSummary(summary[0], summary[1], summary[2]);
1438      }
1439    
1440      final private static PathFilter DEFAULT_FILTER = new PathFilter() {
1441          public boolean accept(Path file) {
1442            return true;
1443          }     
1444        };
1445        
1446      /**
1447       * List the statuses of the files/directories in the given path if the path is
1448       * a directory.
1449       * 
1450       * @param f given path
1451       * @return the statuses of the files/directories in the given patch
1452       * @throws FileNotFoundException when the path does not exist;
1453       *         IOException see specific implementation
1454       */
1455      public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, 
1456                                                             IOException;
1457        
1458      /*
1459       * Filter files/directories in the given path using the user-supplied path
1460       * filter. Results are added to the given array <code>results</code>.
1461       */
1462      private void listStatus(ArrayList<FileStatus> results, Path f,
1463          PathFilter filter) throws FileNotFoundException, IOException {
1464        FileStatus listing[] = listStatus(f);
1465        if (listing == null) {
1466          throw new IOException("Error accessing " + f);
1467        }
1468    
1469        for (int i = 0; i < listing.length; i++) {
1470          if (filter.accept(listing[i].getPath())) {
1471            results.add(listing[i]);
1472          }
1473        }
1474      }
1475    
1476      /**
1477       * @return an iterator over the corrupt files under the given path
1478       * (may contain duplicates if a file has more than one corrupt block)
1479       * @throws IOException
1480       */
1481      public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1482        throws IOException {
1483        throw new UnsupportedOperationException(getClass().getCanonicalName() +
1484                                                " does not support" +
1485                                                " listCorruptFileBlocks");
1486      }
1487    
1488      /**
1489       * Filter files/directories in the given path using the user-supplied path
1490       * filter.
1491       * 
1492       * @param f
1493       *          a path name
1494       * @param filter
1495       *          the user-supplied path filter
1496       * @return an array of FileStatus objects for the files under the given path
1497       *         after applying the filter
1498       * @throws FileNotFoundException when the path does not exist;
1499       *         IOException see specific implementation   
1500       */
1501      public FileStatus[] listStatus(Path f, PathFilter filter) 
1502                                       throws FileNotFoundException, IOException {
1503        ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1504        listStatus(results, f, filter);
1505        return results.toArray(new FileStatus[results.size()]);
1506      }
1507    
1508      /**
1509       * Filter files/directories in the given list of paths using default
1510       * path filter.
1511       * 
1512       * @param files
1513       *          a list of paths
1514       * @return a list of statuses for the files under the given paths after
1515       *         applying the filter default Path filter
1516       * @throws FileNotFoundException when the path does not exist;
1517       *         IOException see specific implementation
1518       */
1519      public FileStatus[] listStatus(Path[] files)
1520          throws FileNotFoundException, IOException {
1521        return listStatus(files, DEFAULT_FILTER);
1522      }
1523    
1524      /**
1525       * Filter files/directories in the given list of paths using user-supplied
1526       * path filter.
1527       * 
1528       * @param files
1529       *          a list of paths
1530       * @param filter
1531       *          the user-supplied path filter
1532       * @return a list of statuses for the files under the given paths after
1533       *         applying the filter
1534       * @throws FileNotFoundException when the path does not exist;
1535       *         IOException see specific implementation
1536       */
1537      public FileStatus[] listStatus(Path[] files, PathFilter filter)
1538          throws FileNotFoundException, IOException {
1539        ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1540        for (int i = 0; i < files.length; i++) {
1541          listStatus(results, files[i], filter);
1542        }
1543        return results.toArray(new FileStatus[results.size()]);
1544      }
1545    
1546      /**
1547       * <p>Return all the files that match filePattern and are not checksum
1548       * files. Results are sorted by their names.
1549       * 
1550       * <p>
1551       * A filename pattern is composed of <i>regular</i> characters and
1552       * <i>special pattern matching</i> characters, which are:
1553       *
1554       * <dl>
1555       *  <dd>
1556       *   <dl>
1557       *    <p>
1558       *    <dt> <tt> ? </tt>
1559       *    <dd> Matches any single character.
1560       *
1561       *    <p>
1562       *    <dt> <tt> * </tt>
1563       *    <dd> Matches zero or more characters.
1564       *
1565       *    <p>
1566       *    <dt> <tt> [<i>abc</i>] </tt>
1567       *    <dd> Matches a single character from character set
1568       *     <tt>{<i>a,b,c</i>}</tt>.
1569       *
1570       *    <p>
1571       *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1572       *    <dd> Matches a single character from the character range
1573       *     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
1574       *     lexicographically less than or equal to character <tt><i>b</i></tt>.
1575       *
1576       *    <p>
1577       *    <dt> <tt> [^<i>a</i>] </tt>
1578       *    <dd> Matches a single character that is not from character set or range
1579       *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
1580       *     immediately to the right of the opening bracket.
1581       *
1582       *    <p>
1583       *    <dt> <tt> \<i>c</i> </tt>
1584       *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
1585       *
1586       *    <p>
1587       *    <dt> <tt> {ab,cd} </tt>
1588       *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1589       *    
1590       *    <p>
1591       *    <dt> <tt> {ab,c{de,fh}} </tt>
1592       *    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
1593       *
1594       *   </dl>
1595       *  </dd>
1596       * </dl>
1597       *
1598       * @param pathPattern a regular expression specifying a pth pattern
1599    
1600       * @return an array of paths that match the path pattern
1601       * @throws IOException
1602       */
1603      public FileStatus[] globStatus(Path pathPattern) throws IOException {
1604        return globStatus(pathPattern, DEFAULT_FILTER);
1605      }
1606      
1607      /**
1608       * Return an array of FileStatus objects whose path names match pathPattern
1609       * and is accepted by the user-supplied path filter. Results are sorted by
1610       * their path names.
1611       * Return null if pathPattern has no glob and the path does not exist.
1612       * Return an empty array if pathPattern has a glob and no path matches it. 
1613       * 
1614       * @param pathPattern
1615       *          a regular expression specifying the path pattern
1616       * @param filter
1617       *          a user-supplied path filter
1618       * @return an array of FileStatus objects
1619       * @throws IOException if any I/O error occurs when fetching file status
1620       */
1621      public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
1622          throws IOException {
1623        String filename = pathPattern.toUri().getPath();
1624        List<FileStatus> allMatches = null;
1625        
1626        List<String> filePatterns = GlobExpander.expand(filename);
1627        for (String filePattern : filePatterns) {
1628          Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern);
1629          List<FileStatus> matches = globStatusInternal(path, filter);
1630          if (matches != null) {
1631            if (allMatches == null) {
1632              allMatches = matches;
1633            } else {
1634              allMatches.addAll(matches);
1635            }
1636          }
1637        }
1638        
1639        FileStatus[] results = null;
1640        if (allMatches != null) {
1641          results = allMatches.toArray(new FileStatus[allMatches.size()]);
1642        } else if (filePatterns.size() > 1) {
1643          // no matches with multiple expansions is a non-matching glob 
1644          results = new FileStatus[0];
1645        }
1646        return results;
1647      }
1648    
1649      // sort gripes because FileStatus Comparable isn't parameterized...
1650      @SuppressWarnings("unchecked") 
1651      private List<FileStatus> globStatusInternal(Path pathPattern,
1652          PathFilter filter) throws IOException {
1653        boolean patternHasGlob = false;       // pathPattern has any globs
1654        List<FileStatus> matches = new ArrayList<FileStatus>();
1655    
1656        // determine starting point
1657        int level = 0;
1658        String baseDir = Path.CUR_DIR;
1659        if (pathPattern.isAbsolute()) {
1660          level = 1; // need to skip empty item at beginning of split list
1661          baseDir = Path.SEPARATOR;
1662        }
1663        
1664        // parse components and determine if it's a glob
1665        String[] components = null;
1666        GlobFilter[] filters = null;
1667        String filename = pathPattern.toUri().getPath();
1668        if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) {
1669          components = filename.split(Path.SEPARATOR);
1670          filters = new GlobFilter[components.length];
1671          for (int i=level; i < components.length; i++) {
1672            filters[i] = new GlobFilter(components[i]);
1673            patternHasGlob |= filters[i].hasPattern();
1674          }
1675          if (!patternHasGlob) {
1676            baseDir = unquotePathComponent(filename);
1677            components = null; // short through to filter check
1678          }
1679        }
1680        
1681        // seed the parent directory path, return if it doesn't exist
1682        try {
1683          matches.add(getFileStatus(new Path(baseDir)));
1684        } catch (FileNotFoundException e) {
1685          return patternHasGlob ? matches : null;
1686        }
1687        
1688        // skip if there are no components other than the basedir
1689        if (components != null) {
1690          // iterate through each path component
1691          for (int i=level; (i < components.length) && !matches.isEmpty(); i++) {
1692            List<FileStatus> children = new ArrayList<FileStatus>();
1693            for (FileStatus match : matches) {
1694              // don't look for children in a file matched by a glob
1695              if (!match.isDirectory()) {
1696                continue;
1697              }
1698              try {
1699                if (filters[i].hasPattern()) {
1700                  // get all children matching the filter
1701                  FileStatus[] statuses = listStatus(match.getPath(), filters[i]);
1702                  children.addAll(Arrays.asList(statuses));
1703                } else {
1704                  // the component does not have a pattern
1705                  String component = unquotePathComponent(components[i]);
1706                  Path child = new Path(match.getPath(), component);
1707                  children.add(getFileStatus(child));
1708                }
1709              } catch (FileNotFoundException e) {
1710                // don't care
1711              }
1712            }
1713            matches = children;
1714          }
1715        }
1716        // remove anything that didn't match the filter
1717        if (!matches.isEmpty()) {
1718          Iterator<FileStatus> iter = matches.iterator();
1719          while (iter.hasNext()) {
1720            if (!filter.accept(iter.next().getPath())) {
1721              iter.remove();
1722            }
1723          }
1724        }
1725        // no final paths, if there were any globs return empty list
1726        if (matches.isEmpty()) {
1727          return patternHasGlob ? matches : null;
1728        }
1729        Collections.sort(matches);
1730        return matches;
1731      }
1732    
1733      /**
1734       * The glob filter builds a regexp per path component.  If the component
1735       * does not contain a shell metachar, then it falls back to appending the
1736       * raw string to the list of built up paths.  This raw path needs to have
1737       * the quoting removed.  Ie. convert all occurances of "\X" to "X"
1738       * @param name of the path component
1739       * @return the unquoted path component
1740       */
1741      private String unquotePathComponent(String name) {
1742        return name.replaceAll("\\\\(.)", "$1");
1743      }
1744      
1745      /**
1746       * List the statuses of the files/directories in the given path if the path is
1747       * a directory. 
1748       * Return the file's status and block locations If the path is a file.
1749       * 
1750       * If a returned status is a file, it contains the file's block locations.
1751       * 
1752       * @param f is the path
1753       *
1754       * @return an iterator that traverses statuses of the files/directories 
1755       *         in the given path
1756       *
1757       * @throws FileNotFoundException If <code>f</code> does not exist
1758       * @throws IOException If an I/O error occurred
1759       */
1760      public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
1761      throws FileNotFoundException, IOException {
1762        return listLocatedStatus(f, DEFAULT_FILTER);
1763      }
1764    
1765      /**
1766       * Listing a directory
1767       * The returned results include its block location if it is a file
1768       * The results are filtered by the given path filter
1769       * @param f a path
1770       * @param filter a path filter
1771       * @return an iterator that traverses statuses of the files/directories 
1772       *         in the given path
1773       * @throws FileNotFoundException if <code>f</code> does not exist
1774       * @throws IOException if any I/O error occurred
1775       */
1776      protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
1777          final PathFilter filter)
1778      throws FileNotFoundException, IOException {
1779        return new RemoteIterator<LocatedFileStatus>() {
1780          private final FileStatus[] stats = listStatus(f, filter);
1781          private int i = 0;
1782    
1783          @Override
1784          public boolean hasNext() {
1785            return i<stats.length;
1786          }
1787    
1788          @Override
1789          public LocatedFileStatus next() throws IOException {
1790            if (!hasNext()) {
1791              throw new NoSuchElementException("No more entry in " + f);
1792            }
1793            FileStatus result = stats[i++];
1794            BlockLocation[] locs = result.isFile() ?
1795                getFileBlockLocations(result.getPath(), 0, result.getLen()) :
1796                null;
1797            return new LocatedFileStatus(result, locs);
1798          }
1799        };
1800      }
1801    
1802      /**
1803       * List the statuses and block locations of the files in the given path.
1804       * 
1805       * If the path is a directory, 
1806       *   if recursive is false, returns files in the directory;
1807       *   if recursive is true, return files in the subtree rooted at the path.
1808       * If the path is a file, return the file's status and block locations.
1809       * 
1810       * @param f is the path
1811       * @param recursive if the subdirectories need to be traversed recursively
1812       *
1813       * @return an iterator that traverses statuses of the files
1814       *
1815       * @throws FileNotFoundException when the path does not exist;
1816       *         IOException see specific implementation
1817       */
1818      public RemoteIterator<LocatedFileStatus> listFiles(
1819          final Path f, final boolean recursive)
1820      throws FileNotFoundException, IOException {
1821        return new RemoteIterator<LocatedFileStatus>() {
1822          private Stack<RemoteIterator<LocatedFileStatus>> itors = 
1823            new Stack<RemoteIterator<LocatedFileStatus>>();
1824          private RemoteIterator<LocatedFileStatus> curItor =
1825            listLocatedStatus(f);
1826          private LocatedFileStatus curFile;
1827         
1828          @Override
1829          public boolean hasNext() throws IOException {
1830            while (curFile == null) {
1831              if (curItor.hasNext()) {
1832                handleFileStat(curItor.next());
1833              } else if (!itors.empty()) {
1834                curItor = itors.pop();
1835              } else {
1836                return false;
1837              }
1838            }
1839            return true;
1840          }
1841    
1842          /**
1843           * Process the input stat.
1844           * If it is a file, return the file stat.
1845           * If it is a directory, traverse the directory if recursive is true;
1846           * ignore it if recursive is false.
1847           * @param stat input status
1848           * @throws IOException if any IO error occurs
1849           */
1850          private void handleFileStat(LocatedFileStatus stat) throws IOException {
1851            if (stat.isFile()) { // file
1852              curFile = stat;
1853            } else if (recursive) { // directory
1854              itors.push(curItor);
1855              curItor = listLocatedStatus(stat.getPath());
1856            }
1857          }
1858    
1859          @Override
1860          public LocatedFileStatus next() throws IOException {
1861            if (hasNext()) {
1862              LocatedFileStatus result = curFile;
1863              curFile = null;
1864              return result;
1865            } 
1866            throw new java.util.NoSuchElementException("No more entry in " + f);
1867          }
1868        };
1869      }
1870      
1871      /** Return the current user's home directory in this filesystem.
1872       * The default implementation returns "/user/$USER/".
1873       */
1874      public Path getHomeDirectory() {
1875        return this.makeQualified(
1876            new Path("/user/"+System.getProperty("user.name")));
1877      }
1878    
1879    
1880      /**
1881       * Set the current working directory for the given file system. All relative
1882       * paths will be resolved relative to it.
1883       * 
1884       * @param new_dir
1885       */
1886      public abstract void setWorkingDirectory(Path new_dir);
1887        
1888      /**
1889       * Get the current working directory for the given file system
1890       * @return the directory pathname
1891       */
1892      public abstract Path getWorkingDirectory();
1893      
1894      
1895      /**
1896       * Note: with the new FilesContext class, getWorkingDirectory()
1897       * will be removed. 
1898       * The working directory is implemented in FilesContext.
1899       * 
1900       * Some file systems like LocalFileSystem have an initial workingDir
1901       * that we use as the starting workingDir. For other file systems
1902       * like HDFS there is no built in notion of an inital workingDir.
1903       * 
1904       * @return if there is built in notion of workingDir then it
1905       * is returned; else a null is returned.
1906       */
1907      protected Path getInitialWorkingDirectory() {
1908        return null;
1909      }
1910    
1911      /**
1912       * Call {@link #mkdirs(Path, FsPermission)} with default permission.
1913       */
1914      public boolean mkdirs(Path f) throws IOException {
1915        return mkdirs(f, FsPermission.getDirDefault());
1916      }
1917    
1918      /**
1919       * Make the given file and all non-existent parents into
1920       * directories. Has the semantics of Unix 'mkdir -p'.
1921       * Existence of the directory hierarchy is not an error.
1922       * @param f path to create
1923       * @param permission to apply to f
1924       */
1925      public abstract boolean mkdirs(Path f, FsPermission permission
1926          ) throws IOException;
1927    
1928      /**
1929       * The src file is on the local disk.  Add it to FS at
1930       * the given dst name and the source is kept intact afterwards
1931       * @param src path
1932       * @param dst path
1933       */
1934      public void copyFromLocalFile(Path src, Path dst)
1935        throws IOException {
1936        copyFromLocalFile(false, src, dst);
1937      }
1938    
1939      /**
1940       * The src files is on the local disk.  Add it to FS at
1941       * the given dst name, removing the source afterwards.
1942       * @param srcs path
1943       * @param dst path
1944       */
1945      public void moveFromLocalFile(Path[] srcs, Path dst)
1946        throws IOException {
1947        copyFromLocalFile(true, true, srcs, dst);
1948      }
1949    
1950      /**
1951       * The src file is on the local disk.  Add it to FS at
1952       * the given dst name, removing the source afterwards.
1953       * @param src path
1954       * @param dst path
1955       */
1956      public void moveFromLocalFile(Path src, Path dst)
1957        throws IOException {
1958        copyFromLocalFile(true, src, dst);
1959      }
1960    
1961      /**
1962       * The src file is on the local disk.  Add it to FS at
1963       * the given dst name.
1964       * delSrc indicates if the source should be removed
1965       * @param delSrc whether to delete the src
1966       * @param src path
1967       * @param dst path
1968       */
1969      public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
1970        throws IOException {
1971        copyFromLocalFile(delSrc, true, src, dst);
1972      }
1973      
1974      /**
1975       * The src files are on the local disk.  Add it to FS at
1976       * the given dst name.
1977       * delSrc indicates if the source should be removed
1978       * @param delSrc whether to delete the src
1979       * @param overwrite whether to overwrite an existing file
1980       * @param srcs array of paths which are source
1981       * @param dst path
1982       */
1983      public void copyFromLocalFile(boolean delSrc, boolean overwrite, 
1984                                    Path[] srcs, Path dst)
1985        throws IOException {
1986        Configuration conf = getConf();
1987        FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
1988      }
1989      
1990      /**
1991       * The src file is on the local disk.  Add it to FS at
1992       * the given dst name.
1993       * delSrc indicates if the source should be removed
1994       * @param delSrc whether to delete the src
1995       * @param overwrite whether to overwrite an existing file
1996       * @param src path
1997       * @param dst path
1998       */
1999      public void copyFromLocalFile(boolean delSrc, boolean overwrite, 
2000                                    Path src, Path dst)
2001        throws IOException {
2002        Configuration conf = getConf();
2003        FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf);
2004      }
2005        
2006      /**
2007       * The src file is under FS, and the dst is on the local disk.
2008       * Copy it from FS control to the local dst name.
2009       * @param src path
2010       * @param dst path
2011       */
2012      public void copyToLocalFile(Path src, Path dst) throws IOException {
2013        copyToLocalFile(false, src, dst);
2014      }
2015        
2016      /**
2017       * The src file is under FS, and the dst is on the local disk.
2018       * Copy it from FS control to the local dst name.
2019       * Remove the source afterwards
2020       * @param src path
2021       * @param dst path
2022       */
2023      public void moveToLocalFile(Path src, Path dst) throws IOException {
2024        copyToLocalFile(true, src, dst);
2025      }
2026    
2027      /**
2028       * The src file is under FS, and the dst is on the local disk.
2029       * Copy it from FS control to the local dst name.
2030       * delSrc indicates if the src will be removed or not.
2031       * @param delSrc whether to delete the src
2032       * @param src path
2033       * @param dst path
2034       */   
2035      public void copyToLocalFile(boolean delSrc, Path src, Path dst)
2036        throws IOException {
2037        copyToLocalFile(delSrc, src, dst, false);
2038      }
2039      
2040        /**
2041       * The src file is under FS, and the dst is on the local disk. Copy it from FS
2042       * control to the local dst name. delSrc indicates if the src will be removed
2043       * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
2044       * as local file system or not. RawLocalFileSystem is non crc file system.So,
2045       * It will not create any crc files at local.
2046       * 
2047       * @param delSrc
2048       *          whether to delete the src
2049       * @param src
2050       *          path
2051       * @param dst
2052       *          path
2053       * @param useRawLocalFileSystem
2054       *          whether to use RawLocalFileSystem as local file system or not.
2055       * 
2056       * @throws IOException
2057       *           - if any IO error
2058       */
2059      public void copyToLocalFile(boolean delSrc, Path src, Path dst,
2060          boolean useRawLocalFileSystem) throws IOException {
2061        Configuration conf = getConf();
2062        FileSystem local = null;
2063        if (useRawLocalFileSystem) {
2064          local = getLocal(conf).getRawFileSystem();
2065        } else {
2066          local = getLocal(conf);
2067        }
2068        FileUtil.copy(this, src, local, dst, delSrc, conf);
2069      }
2070    
2071      /**
2072       * Returns a local File that the user can write output to.  The caller
2073       * provides both the eventual FS target name and the local working
2074       * file.  If the FS is local, we write directly into the target.  If
2075       * the FS is remote, we write into the tmp local area.
2076       * @param fsOutputFile path of output file
2077       * @param tmpLocalFile path of local tmp file
2078       */
2079      public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
2080        throws IOException {
2081        return tmpLocalFile;
2082      }
2083    
2084      /**
2085       * Called when we're all done writing to the target.  A local FS will
2086       * do nothing, because we've written to exactly the right place.  A remote
2087       * FS will copy the contents of tmpLocalFile to the correct target at
2088       * fsOutputFile.
2089       * @param fsOutputFile path of output file
2090       * @param tmpLocalFile path to local tmp file
2091       */
2092      public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
2093        throws IOException {
2094        moveFromLocalFile(tmpLocalFile, fsOutputFile);
2095      }
2096    
2097      /**
2098       * No more filesystem operations are needed.  Will
2099       * release any held locks.
2100       */
2101      public void close() throws IOException {
2102        // delete all files that were marked as delete-on-exit.
2103        processDeleteOnExit();
2104        CACHE.remove(this.key, this);
2105      }
2106    
2107      /** Return the total size of all files in the filesystem.*/
2108      public long getUsed() throws IOException{
2109        long used = 0;
2110        FileStatus[] files = listStatus(new Path("/"));
2111        for(FileStatus file:files){
2112          used += file.getLen();
2113        }
2114        return used;
2115      }
2116      
2117      /**
2118       * Get the block size for a particular file.
2119       * @param f the filename
2120       * @return the number of bytes in a block
2121       */
2122      /** @deprecated Use getFileStatus() instead */
2123      @Deprecated
2124      public long getBlockSize(Path f) throws IOException {
2125        return getFileStatus(f).getBlockSize();
2126      }
2127    
2128      /**
2129       * Return the number of bytes that large input files should be optimally
2130       * be split into to minimize i/o time.
2131       * @deprecated use {@link #getDefaultBlockSize(Path)} instead
2132       */
2133      @Deprecated
2134      public long getDefaultBlockSize() {
2135        // default to 32MB: large enough to minimize the impact of seeks
2136        return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
2137      }
2138        
2139      /** Return the number of bytes that large input files should be optimally
2140       * be split into to minimize i/o time.  The given path will be used to
2141       * locate the actual filesystem.  The full path does not have to exist.
2142       * @param f path of file
2143       * @return the default block size for the path's filesystem
2144       */
2145      public long getDefaultBlockSize(Path f) {
2146        return getDefaultBlockSize();
2147      }
2148    
2149      /**
2150       * Get the default replication.
2151       * @deprecated use {@link #getDefaultReplication(Path)} instead
2152       */
2153      @Deprecated
2154      public short getDefaultReplication() { return 1; }
2155    
2156      /**
2157       * Get the default replication for a path.   The given path will be used to
2158       * locate the actual filesystem.  The full path does not have to exist.
2159       * @param path of the file
2160       * @return default replication for the path's filesystem 
2161       */
2162      public short getDefaultReplication(Path path) {
2163        return getDefaultReplication();
2164      }
2165      
2166      /**
2167       * Return a file status object that represents the path.
2168       * @param f The path we want information from
2169       * @return a FileStatus object
2170       * @throws FileNotFoundException when the path does not exist;
2171       *         IOException see specific implementation
2172       */
2173      public abstract FileStatus getFileStatus(Path f) throws IOException;
2174    
2175      /**
2176       * Get the checksum of a file.
2177       *
2178       * @param f The file path
2179       * @return The file checksum.  The default return value is null,
2180       *  which indicates that no checksum algorithm is implemented
2181       *  in the corresponding FileSystem.
2182       */
2183      public FileChecksum getFileChecksum(Path f) throws IOException {
2184        return null;
2185      }
2186      
2187      /**
2188       * Set the verify checksum flag. This is only applicable if the 
2189       * corresponding FileSystem supports checksum. By default doesn't do anything.
2190       * @param verifyChecksum
2191       */
2192      public void setVerifyChecksum(boolean verifyChecksum) {
2193        //doesn't do anything
2194      }
2195    
2196      /**
2197       * Set the write checksum flag. This is only applicable if the 
2198       * corresponding FileSystem supports checksum. By default doesn't do anything.
2199       * @param writeChecksum
2200       */
2201      public void setWriteChecksum(boolean writeChecksum) {
2202        //doesn't do anything
2203      }
2204    
2205      /**
2206       * Returns a status object describing the use and capacity of the
2207       * file system. If the file system has multiple partitions, the
2208       * use and capacity of the root partition is reflected.
2209       * 
2210       * @return a FsStatus object
2211       * @throws IOException
2212       *           see specific implementation
2213       */
2214      public FsStatus getStatus() throws IOException {
2215        return getStatus(null);
2216      }
2217    
2218      /**
2219       * Returns a status object describing the use and capacity of the
2220       * file system. If the file system has multiple partitions, the
2221       * use and capacity of the partition pointed to by the specified
2222       * path is reflected.
2223       * @param p Path for which status should be obtained. null means
2224       * the default partition. 
2225       * @return a FsStatus object
2226       * @throws IOException
2227       *           see specific implementation
2228       */
2229      public FsStatus getStatus(Path p) throws IOException {
2230        return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE);
2231      }
2232    
2233      /**
2234       * Set permission of a path.
2235       * @param p
2236       * @param permission
2237       */
2238      public void setPermission(Path p, FsPermission permission
2239          ) throws IOException {
2240      }
2241    
2242      /**
2243       * Set owner of a path (i.e. a file or a directory).
2244       * The parameters username and groupname cannot both be null.
2245       * @param p The path
2246       * @param username If it is null, the original username remains unchanged.
2247       * @param groupname If it is null, the original groupname remains unchanged.
2248       */
2249      public void setOwner(Path p, String username, String groupname
2250          ) throws IOException {
2251      }
2252    
2253      /**
2254       * Set access time of a file
2255       * @param p The path
2256       * @param mtime Set the modification time of this file.
2257       *              The number of milliseconds since Jan 1, 1970. 
2258       *              A value of -1 means that this call should not set modification time.
2259       * @param atime Set the access time of this file.
2260       *              The number of milliseconds since Jan 1, 1970. 
2261       *              A value of -1 means that this call should not set access time.
2262       */
2263      public void setTimes(Path p, long mtime, long atime
2264          ) throws IOException {
2265      }
2266    
2267      // making it volatile to be able to do a double checked locking
2268      private volatile static boolean FILE_SYSTEMS_LOADED = false;
2269    
2270      private static final Map<String, Class<? extends FileSystem>>
2271        SERVICE_FILE_SYSTEMS = new HashMap<String, Class<? extends FileSystem>>();
2272    
2273      private static void loadFileSystems() {
2274        synchronized (FileSystem.class) {
2275          if (!FILE_SYSTEMS_LOADED) {
2276            ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
2277            for (FileSystem fs : serviceLoader) {
2278              SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
2279            }
2280            FILE_SYSTEMS_LOADED = true;
2281          }
2282        }
2283      }
2284    
2285      public static Class<? extends FileSystem> getFileSystemClass(String scheme,
2286          Configuration conf) throws IOException {
2287        if (!FILE_SYSTEMS_LOADED) {
2288          loadFileSystems();
2289        }
2290        Class<? extends FileSystem> clazz = null;
2291        if (conf != null) {
2292          clazz = (Class<? extends FileSystem>) conf.getClass("fs." + scheme + ".impl", null);
2293        }
2294        if (clazz == null) {
2295          clazz = SERVICE_FILE_SYSTEMS.get(scheme);
2296        }
2297        if (clazz == null) {
2298          throw new IOException("No FileSystem for scheme: " + scheme);
2299        }
2300        return clazz;
2301      }
2302    
2303      private static FileSystem createFileSystem(URI uri, Configuration conf
2304          ) throws IOException {
2305        Class<?> clazz = getFileSystemClass(uri.getScheme(), conf);
2306        if (clazz == null) {
2307          throw new IOException("No FileSystem for scheme: " + uri.getScheme());
2308        }
2309        FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
2310        fs.initialize(uri, conf);
2311        return fs;
2312      }
2313    
2314      /** Caching FileSystem objects */
2315      static class Cache {
2316        private final ClientFinalizer clientFinalizer = new ClientFinalizer();
2317    
2318        private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>();
2319        private final Set<Key> toAutoClose = new HashSet<Key>();
2320    
2321        /** A variable that makes all objects in the cache unique */
2322        private static AtomicLong unique = new AtomicLong(1);
2323    
2324        FileSystem get(URI uri, Configuration conf) throws IOException{
2325          Key key = new Key(uri, conf);
2326          return getInternal(uri, conf, key);
2327        }
2328    
2329        /** The objects inserted into the cache using this method are all unique */
2330        FileSystem getUnique(URI uri, Configuration conf) throws IOException{
2331          Key key = new Key(uri, conf, unique.getAndIncrement());
2332          return getInternal(uri, conf, key);
2333        }
2334    
2335        private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{
2336          FileSystem fs;
2337          synchronized (this) {
2338            fs = map.get(key);
2339          }
2340          if (fs != null) {
2341            return fs;
2342          }
2343    
2344          fs = createFileSystem(uri, conf);
2345          synchronized (this) { // refetch the lock again
2346            FileSystem oldfs = map.get(key);
2347            if (oldfs != null) { // a file system is created while lock is releasing
2348              fs.close(); // close the new file system
2349              return oldfs;  // return the old file system
2350            }
2351            
2352            // now insert the new file system into the map
2353            if (map.isEmpty()
2354                    && !ShutdownHookManager.get().isShutdownInProgress()) {
2355              ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY);
2356            }
2357            fs.key = key;
2358            map.put(key, fs);
2359            if (conf.getBoolean("fs.automatic.close", true)) {
2360              toAutoClose.add(key);
2361            }
2362            return fs;
2363          }
2364        }
2365    
2366        synchronized void remove(Key key, FileSystem fs) {
2367          if (map.containsKey(key) && fs == map.get(key)) {
2368            map.remove(key);
2369            toAutoClose.remove(key);
2370            }
2371        }
2372    
2373        synchronized void closeAll() throws IOException {
2374          closeAll(false);
2375        }
2376    
2377        /**
2378         * Close all FileSystem instances in the Cache.
2379         * @param onlyAutomatic only close those that are marked for automatic closing
2380         */
2381        synchronized void closeAll(boolean onlyAutomatic) throws IOException {
2382          List<IOException> exceptions = new ArrayList<IOException>();
2383    
2384          // Make a copy of the keys in the map since we'll be modifying
2385          // the map while iterating over it, which isn't safe.
2386          List<Key> keys = new ArrayList<Key>();
2387          keys.addAll(map.keySet());
2388    
2389          for (Key key : keys) {
2390            final FileSystem fs = map.get(key);
2391    
2392            if (onlyAutomatic && !toAutoClose.contains(key)) {
2393              continue;
2394            }
2395    
2396            //remove from cache
2397            remove(key, fs);
2398    
2399            if (fs != null) {
2400              try {
2401                fs.close();
2402              }
2403              catch(IOException ioe) {
2404                exceptions.add(ioe);
2405              }
2406            }
2407          }
2408    
2409          if (!exceptions.isEmpty()) {
2410            throw MultipleIOException.createIOException(exceptions);
2411          }
2412        }
2413    
2414        private class ClientFinalizer implements Runnable {
2415          public synchronized void run() {
2416            try {
2417              closeAll(true);
2418            } catch (IOException e) {
2419              LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e);
2420            }
2421          }
2422        }
2423    
2424        synchronized void closeAll(UserGroupInformation ugi) throws IOException {
2425          List<FileSystem> targetFSList = new ArrayList<FileSystem>();
2426          //Make a pass over the list and collect the filesystems to close
2427          //we cannot close inline since close() removes the entry from the Map
2428          for (Map.Entry<Key, FileSystem> entry : map.entrySet()) {
2429            final Key key = entry.getKey();
2430            final FileSystem fs = entry.getValue();
2431            if (ugi.equals(key.ugi) && fs != null) {
2432              targetFSList.add(fs);   
2433            }
2434          }
2435          List<IOException> exceptions = new ArrayList<IOException>();
2436          //now make a pass over the target list and close each
2437          for (FileSystem fs : targetFSList) {
2438            try {
2439              fs.close();
2440            }
2441            catch(IOException ioe) {
2442              exceptions.add(ioe);
2443            }
2444          }
2445          if (!exceptions.isEmpty()) {
2446            throw MultipleIOException.createIOException(exceptions);
2447          }
2448        }
2449    
2450        /** FileSystem.Cache.Key */
2451        static class Key {
2452          final String scheme;
2453          final String authority;
2454          final UserGroupInformation ugi;
2455          final long unique;   // an artificial way to make a key unique
2456    
2457          Key(URI uri, Configuration conf) throws IOException {
2458            this(uri, conf, 0);
2459          }
2460    
2461          Key(URI uri, Configuration conf, long unique) throws IOException {
2462            scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase();
2463            authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase();
2464            this.unique = unique;
2465            
2466            this.ugi = UserGroupInformation.getCurrentUser();
2467          }
2468    
2469          /** {@inheritDoc} */
2470          public int hashCode() {
2471            return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique;
2472          }
2473    
2474          static boolean isEqual(Object a, Object b) {
2475            return a == b || (a != null && a.equals(b));        
2476          }
2477    
2478          /** {@inheritDoc} */
2479          public boolean equals(Object obj) {
2480            if (obj == this) {
2481              return true;
2482            }
2483            if (obj != null && obj instanceof Key) {
2484              Key that = (Key)obj;
2485              return isEqual(this.scheme, that.scheme)
2486                     && isEqual(this.authority, that.authority)
2487                     && isEqual(this.ugi, that.ugi)
2488                     && (this.unique == that.unique);
2489            }
2490            return false;        
2491          }
2492    
2493          /** {@inheritDoc} */
2494          public String toString() {
2495            return "("+ugi.toString() + ")@" + scheme + "://" + authority;        
2496          }
2497        }
2498      }
2499      
2500      /**
2501       * Tracks statistics about how many reads, writes, and so forth have been
2502       * done in a FileSystem.
2503       * 
2504       * Since there is only one of these objects per FileSystem, there will 
2505       * typically be many threads writing to this object.  Almost every operation
2506       * on an open file will involve a write to this object.  In contrast, reading
2507       * statistics is done infrequently by most programs, and not at all by others.
2508       * Hence, this is optimized for writes.
2509       * 
2510       * Each thread writes to its own thread-local area of memory.  This removes 
2511       * contention and allows us to scale up to many, many threads.  To read
2512       * statistics, the reader thread totals up the contents of all of the 
2513       * thread-local data areas.
2514       */
2515      public static final class Statistics {
2516        /**
2517         * Statistics data.
2518         * 
2519         * There is only a single writer to thread-local StatisticsData objects.
2520         * Hence, volatile is adequate here-- we do not need AtomicLong or similar
2521         * to prevent lost updates.
2522         * The Java specification guarantees that updates to volatile longs will
2523         * be perceived as atomic with respect to other threads, which is all we
2524         * need.
2525         */
2526        private static class StatisticsData {
2527          volatile long bytesRead;
2528          volatile long bytesWritten;
2529          volatile int readOps;
2530          volatile int largeReadOps;
2531          volatile int writeOps;
2532          /**
2533           * Stores a weak reference to the thread owning this StatisticsData.
2534           * This allows us to remove StatisticsData objects that pertain to
2535           * threads that no longer exist.
2536           */
2537          final WeakReference<Thread> owner;
2538    
2539          StatisticsData(WeakReference<Thread> owner) {
2540            this.owner = owner;
2541          }
2542    
2543          /**
2544           * Add another StatisticsData object to this one.
2545           */
2546          void add(StatisticsData other) {
2547            this.bytesRead += other.bytesRead;
2548            this.bytesWritten += other.bytesWritten;
2549            this.readOps += other.readOps;
2550            this.largeReadOps += other.largeReadOps;
2551            this.writeOps += other.writeOps;
2552          }
2553    
2554          /**
2555           * Negate the values of all statistics.
2556           */
2557          void negate() {
2558            this.bytesRead = -this.bytesRead;
2559            this.bytesWritten = -this.bytesWritten;
2560            this.readOps = -this.readOps;
2561            this.largeReadOps = -this.largeReadOps;
2562            this.writeOps = -this.writeOps;
2563          }
2564    
2565          @Override
2566          public String toString() {
2567            return bytesRead + " bytes read, " + bytesWritten + " bytes written, "
2568                + readOps + " read ops, " + largeReadOps + " large read ops, "
2569                + writeOps + " write ops";
2570          }
2571        }
2572    
2573        private interface StatisticsAggregator<T> {
2574          void accept(StatisticsData data);
2575          T aggregate();
2576        }
2577    
2578        private final String scheme;
2579    
2580        /**
2581         * rootData is data that doesn't belong to any thread, but will be added
2582         * to the totals.  This is useful for making copies of Statistics objects,
2583         * and for storing data that pertains to threads that have been garbage
2584         * collected.  Protected by the Statistics lock.
2585         */
2586        private final StatisticsData rootData;
2587    
2588        /**
2589         * Thread-local data.
2590         */
2591        private final ThreadLocal<StatisticsData> threadData;
2592        
2593        /**
2594         * List of all thread-local data areas.  Protected by the Statistics lock.
2595         */
2596        private LinkedList<StatisticsData> allData;
2597    
2598        public Statistics(String scheme) {
2599          this.scheme = scheme;
2600          this.rootData = new StatisticsData(null);
2601          this.threadData = new ThreadLocal<StatisticsData>();
2602          this.allData = null;
2603        }
2604    
2605        /**
2606         * Copy constructor.
2607         * 
2608         * @param other    The input Statistics object which is cloned.
2609         */
2610        public Statistics(Statistics other) {
2611          this.scheme = other.scheme;
2612          this.rootData = new StatisticsData(null);
2613          other.visitAll(new StatisticsAggregator<Void>() {
2614            @Override
2615            public void accept(StatisticsData data) {
2616              rootData.add(data);
2617            }
2618    
2619            public Void aggregate() {
2620              return null;
2621            }
2622          });
2623          this.threadData = new ThreadLocal<StatisticsData>();
2624        }
2625    
2626        /**
2627         * Get or create the thread-local data associated with the current thread.
2628         */
2629        private StatisticsData getThreadData() {
2630          StatisticsData data = threadData.get();
2631          if (data == null) {
2632            data = new StatisticsData(
2633                new WeakReference<Thread>(Thread.currentThread()));
2634            threadData.set(data);
2635            synchronized(this) {
2636              if (allData == null) {
2637                allData = new LinkedList<StatisticsData>();
2638              }
2639              allData.add(data);
2640            }
2641          }
2642          return data;
2643        }
2644    
2645        /**
2646         * Increment the bytes read in the statistics
2647         * @param newBytes the additional bytes read
2648         */
2649        public void incrementBytesRead(long newBytes) {
2650          getThreadData().bytesRead += newBytes;
2651        }
2652        
2653        /**
2654         * Increment the bytes written in the statistics
2655         * @param newBytes the additional bytes written
2656         */
2657        public void incrementBytesWritten(long newBytes) {
2658          getThreadData().bytesWritten += newBytes;
2659        }
2660        
2661        /**
2662         * Increment the number of read operations
2663         * @param count number of read operations
2664         */
2665        public void incrementReadOps(int count) {
2666          getThreadData().readOps += count;
2667        }
2668    
2669        /**
2670         * Increment the number of large read operations
2671         * @param count number of large read operations
2672         */
2673        public void incrementLargeReadOps(int count) {
2674          getThreadData().largeReadOps += count;
2675        }
2676    
2677        /**
2678         * Increment the number of write operations
2679         * @param count number of write operations
2680         */
2681        public void incrementWriteOps(int count) {
2682          getThreadData().writeOps += count;
2683        }
2684    
2685        /**
2686         * Apply the given aggregator to all StatisticsData objects associated with
2687         * this Statistics object.
2688         *
2689         * For each StatisticsData object, we will call accept on the visitor.
2690         * Finally, at the end, we will call aggregate to get the final total. 
2691         *
2692         * @param         The visitor to use.
2693         * @return        The total.
2694         */
2695        private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
2696          visitor.accept(rootData);
2697          if (allData != null) {
2698            for (Iterator<StatisticsData> iter = allData.iterator();
2699                iter.hasNext(); ) {
2700              StatisticsData data = iter.next();
2701              visitor.accept(data);
2702              if (data.owner.get() == null) {
2703                /*
2704                 * If the thread that created this thread-local data no
2705                 * longer exists, remove the StatisticsData from our list
2706                 * and fold the values into rootData.
2707                 */
2708                rootData.add(data);
2709                iter.remove();
2710              }
2711            }
2712          }
2713          return visitor.aggregate();
2714        }
2715    
2716        /**
2717         * Get the total number of bytes read
2718         * @return the number of bytes
2719         */
2720        public long getBytesRead() {
2721          return visitAll(new StatisticsAggregator<Long>() {
2722            private long bytesRead = 0;
2723    
2724            @Override
2725            public void accept(StatisticsData data) {
2726              bytesRead += data.bytesRead;
2727            }
2728    
2729            public Long aggregate() {
2730              return bytesRead;
2731            }
2732          });
2733        }
2734        
2735        /**
2736         * Get the total number of bytes written
2737         * @return the number of bytes
2738         */
2739        public long getBytesWritten() {
2740          return visitAll(new StatisticsAggregator<Long>() {
2741            private long bytesWritten = 0;
2742    
2743            @Override
2744            public void accept(StatisticsData data) {
2745              bytesWritten += data.bytesWritten;
2746            }
2747    
2748            public Long aggregate() {
2749              return bytesWritten;
2750            }
2751          });
2752        }
2753        
2754        /**
2755         * Get the number of file system read operations such as list files
2756         * @return number of read operations
2757         */
2758        public int getReadOps() {
2759          return visitAll(new StatisticsAggregator<Integer>() {
2760            private int readOps = 0;
2761    
2762            @Override
2763            public void accept(StatisticsData data) {
2764              readOps += data.readOps;
2765              readOps += data.largeReadOps;
2766            }
2767    
2768            public Integer aggregate() {
2769              return readOps;
2770            }
2771          });
2772        }
2773    
2774        /**
2775         * Get the number of large file system read operations such as list files
2776         * under a large directory
2777         * @return number of large read operations
2778         */
2779        public int getLargeReadOps() {
2780          return visitAll(new StatisticsAggregator<Integer>() {
2781            private int largeReadOps = 0;
2782    
2783            @Override
2784            public void accept(StatisticsData data) {
2785              largeReadOps += data.largeReadOps;
2786            }
2787    
2788            public Integer aggregate() {
2789              return largeReadOps;
2790            }
2791          });
2792        }
2793    
2794        /**
2795         * Get the number of file system write operations such as create, append 
2796         * rename etc.
2797         * @return number of write operations
2798         */
2799        public int getWriteOps() {
2800          return visitAll(new StatisticsAggregator<Integer>() {
2801            private int writeOps = 0;
2802    
2803            @Override
2804            public void accept(StatisticsData data) {
2805              writeOps += data.writeOps;
2806            }
2807    
2808            public Integer aggregate() {
2809              return writeOps;
2810            }
2811          });
2812        }
2813    
2814    
2815        @Override
2816        public String toString() {
2817          return visitAll(new StatisticsAggregator<String>() {
2818            private StatisticsData total = new StatisticsData(null);
2819    
2820            @Override
2821            public void accept(StatisticsData data) {
2822              total.add(data);
2823            }
2824    
2825            public String aggregate() {
2826              return total.toString();
2827            }
2828          });
2829        }
2830    
2831        /**
2832         * Resets all statistics to 0.
2833         *
2834         * In order to reset, we add up all the thread-local statistics data, and
2835         * set rootData to the negative of that.
2836         *
2837         * This may seem like a counterintuitive way to reset the statsitics.  Why
2838         * can't we just zero out all the thread-local data?  Well, thread-local
2839         * data can only be modified by the thread that owns it.  If we tried to
2840         * modify the thread-local data from this thread, our modification might get
2841         * interleaved with a read-modify-write operation done by the thread that
2842         * owns the data.  That would result in our update getting lost.
2843         *
2844         * The approach used here avoids this problem because it only ever reads
2845         * (not writes) the thread-local data.  Both reads and writes to rootData
2846         * are done under the lock, so we're free to modify rootData from any thread
2847         * that holds the lock.
2848         */
2849        public void reset() {
2850          visitAll(new StatisticsAggregator<Void>() {
2851            private StatisticsData total = new StatisticsData(null);
2852    
2853            @Override
2854            public void accept(StatisticsData data) {
2855              total.add(data);
2856            }
2857    
2858            public Void aggregate() {
2859              total.negate();
2860              rootData.add(total);
2861              return null;
2862            }
2863          });
2864        }
2865        
2866        /**
2867         * Get the uri scheme associated with this statistics object.
2868         * @return the schema associated with this set of statistics
2869         */
2870        public String getScheme() {
2871          return scheme;
2872        }
2873      }
2874      
2875      /**
2876       * Get the Map of Statistics object indexed by URI Scheme.
2877       * @return a Map having a key as URI scheme and value as Statistics object
2878       * @deprecated use {@link #getAllStatistics} instead
2879       */
2880      @Deprecated
2881      public static synchronized Map<String, Statistics> getStatistics() {
2882        Map<String, Statistics> result = new HashMap<String, Statistics>();
2883        for(Statistics stat: statisticsTable.values()) {
2884          result.put(stat.getScheme(), stat);
2885        }
2886        return result;
2887      }
2888    
2889      /**
2890       * Return the FileSystem classes that have Statistics
2891       */
2892      public static synchronized List<Statistics> getAllStatistics() {
2893        return new ArrayList<Statistics>(statisticsTable.values());
2894      }
2895      
2896      /**
2897       * Get the statistics for a particular file system
2898       * @param cls the class to lookup
2899       * @return a statistics object
2900       */
2901      public static synchronized 
2902      Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) {
2903        Statistics result = statisticsTable.get(cls);
2904        if (result == null) {
2905          result = new Statistics(scheme);
2906          statisticsTable.put(cls, result);
2907        }
2908        return result;
2909      }
2910      
2911      /**
2912       * Reset all statistics for all file systems
2913       */
2914      public static synchronized void clearStatistics() {
2915        for(Statistics stat: statisticsTable.values()) {
2916          stat.reset();
2917        }
2918      }
2919    
2920      /**
2921       * Print all statistics for all file systems
2922       */
2923      public static synchronized
2924      void printStatistics() throws IOException {
2925        for (Map.Entry<Class<? extends FileSystem>, Statistics> pair: 
2926                statisticsTable.entrySet()) {
2927          System.out.println("  FileSystem " + pair.getKey().getName() + 
2928                             ": " + pair.getValue());
2929        }
2930      }
2931    }