001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs;
019    
020    import java.io.FileNotFoundException;
021    import java.io.IOException;
022    import java.io.InputStream;
023    import java.io.OutputStream;
024    import java.net.URI;
025    import java.security.PrivilegedExceptionAction;
026    import java.util.ArrayList;
027    import java.util.Arrays;
028    import java.util.EnumSet;
029    import java.util.HashSet;
030    import java.util.IdentityHashMap;
031    import java.util.List;
032    import java.util.Map;
033    import java.util.Set;
034    import java.util.Stack;
035    import java.util.TreeSet;
036    import java.util.Map.Entry;
037    
038    import org.apache.commons.logging.Log;
039    import org.apache.commons.logging.LogFactory;
040    import org.apache.hadoop.HadoopIllegalArgumentException;
041    import org.apache.hadoop.classification.InterfaceAudience;
042    import org.apache.hadoop.classification.InterfaceStability;
043    import org.apache.hadoop.conf.Configuration;
044    import org.apache.hadoop.fs.FileSystem.Statistics;
045    import org.apache.hadoop.fs.Options.CreateOpts;
046    import org.apache.hadoop.fs.permission.FsPermission;
047    import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
048    import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
049    import org.apache.hadoop.io.IOUtils;
050    import org.apache.hadoop.ipc.RpcClientException;
051    import org.apache.hadoop.ipc.RpcServerException;
052    import org.apache.hadoop.ipc.UnexpectedServerException;
053    import org.apache.hadoop.fs.InvalidPathException;
054    import org.apache.hadoop.security.AccessControlException;
055    import org.apache.hadoop.security.UserGroupInformation;
056    import org.apache.hadoop.security.token.Token;
057    import org.apache.hadoop.util.ShutdownHookManager;
058    
059    /**
060     * The FileContext class provides an interface for users of the Hadoop
061     * file system. It exposes a number of file system operations, e.g. create,
062     * open, list.
063     * 
064     * <h2>Path Names</h2>
065     * 
066     * The Hadoop file system supports a URI namespace and URI names. This enables
067     * multiple types of file systems to be referenced using fully-qualified URIs.
068     * Two common Hadoop file system implementations are
069     * <ul>
070     * <li>the local file system: file:///path
071     * <li>the HDFS file system: hdfs://nnAddress:nnPort/path
072     * </ul>
073     * 
074     * The Hadoop file system also supports additional naming schemes besides URIs.
075     * Hadoop has the concept of a <i>default file system</i>, which implies a
076     * default URI scheme and authority. This enables <i>slash-relative names</i>
077     * relative to the default FS, which are more convenient for users and
078     * application writers. The default FS is typically set by the user's
079     * environment, though it can also be manually specified.
080     * <p>
081     * 
082     * Hadoop also supports <i>working-directory-relative</i> names, which are paths
083     * relative to the current working directory (similar to Unix). The working
084     * directory can be in a different file system than the default FS.
085     * <p>
086     * Thus, Hadoop path names can be specified as one of the following:
087     * <ul>
088     * <li>a fully-qualified URI: scheme://authority/path (e.g.
089     * hdfs://nnAddress:nnPort/foo/bar)
090     * <li>a slash-relative name: path relative to the default file system (e.g.
091     * /foo/bar)
092     * <li>a working-directory-relative name: path relative to the working dir (e.g.
093     * foo/bar)
094     * </ul>
095     *  Relative paths with scheme (scheme:foo/bar) are illegal.
096     *  
097     * <h2>Role of FileContext and Configuration Defaults</h2>
098     *
099     * The FileContext is the analogue of per-process file-related state in Unix. It
100     * contains two properties:
101     * 
102     * <ul>
103     * <li>the default file system (for resolving slash-relative names)
104     * <li>the umask (for file permissions)
105     * </ul>
106     * In general, these properties are obtained from the default configuration file
107     * in the user's environment (see {@link Configuration}).
108     * 
109     * Further file system properties are specified on the server-side. File system
110     * operations default to using these server-side defaults unless otherwise
111     * specified.
112     * <p>
113     * The file system related server-side defaults are:
114     *  <ul>
115     *  <li> the home directory (default is "/user/userName")
116     *  <li> the initial wd (only for local fs)
117     *  <li> replication factor
118     *  <li> block size
119     *  <li> buffer size
120     *  <li> encryptDataTransfer 
121     *  <li> checksum option. (checksumType and  bytesPerChecksum)
122     *  </ul>
123     *
124     * <h2>Example Usage</h2>
125     *
126     * Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
127     *   Unspecified values come from core-defaults.xml in the release jar.
128     *  <ul>  
129     *  <li> myFContext = FileContext.getFileContext(); // uses the default config
130     *                                                // which has your default FS 
131     *  <li>  myFContext.create(path, ...);
132     *  <li>  myFContext.setWorkingDir(path);
133     *  <li>  myFContext.open (path, ...);  
134     *  <li>...
135     *  </ul>  
136     * Example 2: Get a FileContext with a specific URI as the default FS
137     *  <ul>  
138     *  <li> myFContext = FileContext.getFileContext(URI);
139     *  <li> myFContext.create(path, ...);
140     *  <li>...
141     * </ul>
142     * Example 3: FileContext with local file system as the default
143     *  <ul> 
144     *  <li> myFContext = FileContext.getLocalFSFileContext();
145     *  <li> myFContext.create(path, ...);
146     *  <li> ...
147     *  </ul> 
148     * Example 4: Use a specific config, ignoring $HADOOP_CONFIG
149     *  Generally you should not need use a config unless you are doing
150     *   <ul> 
151     *   <li> configX = someConfigSomeOnePassedToYou;
152     *   <li> myFContext = getFileContext(configX); // configX is not changed,
153     *                                              // is passed down 
154     *   <li> myFContext.create(path, ...);
155     *   <li>...
156     *  </ul>                                          
157     *    
158     */
159    
160    @InterfaceAudience.Public
161    @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
162    public final class FileContext {
163      
164      public static final Log LOG = LogFactory.getLog(FileContext.class);
165      /**
166       * Default permission for directory and symlink
167       * In previous versions, this default permission was also used to
168       * create files, so files created end up with ugo+x permission.
169       * See HADOOP-9155 for detail. 
170       * Two new constants are added to solve this, please use 
171       * {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
172       * {@link FileContext#FILE_DEFAULT_PERM} for file.
173       * This constant is kept for compatibility.
174       */
175      public static final FsPermission DEFAULT_PERM = FsPermission.getDefault();
176      /**
177       * Default permission for directory
178       */
179      public static final FsPermission DIR_DEFAULT_PERM = FsPermission.getDirDefault();
180      /**
181       * Default permission for file
182       */
183      public static final FsPermission FILE_DEFAULT_PERM = FsPermission.getFileDefault();
184    
185      /**
186       * Priority of the FileContext shutdown hook.
187       */
188      public static final int SHUTDOWN_HOOK_PRIORITY = 20;
189    
190      /**
191       * List of files that should be deleted on JVM shutdown.
192       */
193      static final Map<FileContext, Set<Path>> DELETE_ON_EXIT = 
194        new IdentityHashMap<FileContext, Set<Path>>();
195    
196      /** JVM shutdown hook thread. */
197      static final FileContextFinalizer FINALIZER = 
198        new FileContextFinalizer();
199      
200      private static final PathFilter DEFAULT_FILTER = new PathFilter() {
201        public boolean accept(final Path file) {
202          return true;
203        }
204      };
205      
206      /**
207       * The FileContext is defined by.
208       *  1) defaultFS (slash)
209       *  2) wd
210       *  3) umask
211       */   
212      private final AbstractFileSystem defaultFS; //default FS for this FileContext.
213      private Path workingDir;          // Fully qualified
214      private FsPermission umask;
215      private final Configuration conf;
216      private final UserGroupInformation ugi;
217    
218      private FileContext(final AbstractFileSystem defFs,
219        final FsPermission theUmask, final Configuration aConf) {
220        defaultFS = defFs;
221        umask = FsPermission.getUMask(aConf);
222        conf = aConf;
223        try {
224          ugi = UserGroupInformation.getCurrentUser();
225        } catch (IOException e) {
226          LOG.error("Exception in getCurrentUser: ",e);
227          throw new RuntimeException("Failed to get the current user " +
228                    "while creating a FileContext", e);
229        }
230        /*
231         * Init the wd.
232         * WorkingDir is implemented at the FileContext layer 
233         * NOT at the AbstractFileSystem layer. 
234         * If the DefaultFS, such as localFilesystem has a notion of
235         *  builtin WD, we use that as the initial WD.
236         *  Otherwise the WD is initialized to the home directory.
237         */
238        workingDir = defaultFS.getInitialWorkingDirectory();
239        if (workingDir == null) {
240          workingDir = defaultFS.getHomeDirectory();
241        }
242        util = new Util(); // for the inner class
243      }
244     
245      /* 
246       * Remove relative part - return "absolute":
247       * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
248       * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
249       * ("/foo/bar") are returned unchanged.
250       * 
251       * Applications that use FileContext should use #makeQualified() since
252       * they really want a fully qualified URI.
253       * Hence this method is not called makeAbsolute() and 
254       * has been deliberately declared private.
255       */
256      private Path fixRelativePart(Path p) {
257        if (p.isUriPathAbsolute()) {
258          return p;
259        } else {
260          return new Path(workingDir, p);
261        }
262      }
263    
264      /**
265       * Delete all the paths that were marked as delete-on-exit.
266       */
267      static void processDeleteOnExit() {
268        synchronized (DELETE_ON_EXIT) {
269          Set<Entry<FileContext, Set<Path>>> set = DELETE_ON_EXIT.entrySet();
270          for (Entry<FileContext, Set<Path>> entry : set) {
271            FileContext fc = entry.getKey();
272            Set<Path> paths = entry.getValue();
273            for (Path path : paths) {
274              try {
275                fc.delete(path, true);
276              } catch (IOException e) {
277                LOG.warn("Ignoring failure to deleteOnExit for path " + path);
278              }
279            }
280          }
281          DELETE_ON_EXIT.clear();
282        }
283      }
284      
285      /**
286       * Pathnames with scheme and relative path are illegal.
287       * @param path to be checked
288       */
289      private static void checkNotSchemeWithRelative(final Path path) {
290        if (path.toUri().isAbsolute() && !path.isUriPathAbsolute()) {
291          throw new HadoopIllegalArgumentException(
292              "Unsupported name: has scheme but relative path-part");
293        }
294      }
295    
296      /**
297       * Get the file system of supplied path.
298       * 
299       * @param absOrFqPath - absolute or fully qualified path
300       * @return the file system of the path
301       * 
302       * @throws UnsupportedFileSystemException If the file system for
303       *           <code>absOrFqPath</code> is not supported.
304       * @throws IOExcepton If the file system for <code>absOrFqPath</code> could
305       *         not be instantiated.
306       */
307      private AbstractFileSystem getFSofPath(final Path absOrFqPath)
308          throws UnsupportedFileSystemException, IOException {
309        checkNotSchemeWithRelative(absOrFqPath);
310        if (!absOrFqPath.isAbsolute() && absOrFqPath.toUri().getScheme() == null) {
311          throw new HadoopIllegalArgumentException(
312              "FileContext Bug: path is relative");
313        }
314    
315        try { 
316          // Is it the default FS for this FileContext?
317          defaultFS.checkPath(absOrFqPath);
318          return defaultFS;
319        } catch (Exception e) { // it is different FileSystem
320          return getAbstractFileSystem(ugi, absOrFqPath.toUri(), conf);
321        }
322      }
323      
324      private static AbstractFileSystem getAbstractFileSystem(
325          UserGroupInformation user, final URI uri, final Configuration conf)
326          throws UnsupportedFileSystemException, IOException {
327        try {
328          return user.doAs(new PrivilegedExceptionAction<AbstractFileSystem>() {
329            public AbstractFileSystem run() throws UnsupportedFileSystemException {
330              return AbstractFileSystem.get(uri, conf);
331            }
332          });
333        } catch (InterruptedException ex) {
334          LOG.error(ex);
335          throw new IOException("Failed to get the AbstractFileSystem for path: "
336              + uri, ex);
337        }
338      }
339      
340      /**
341       * Protected Static Factory methods for getting a FileContexts
342       * that take a AbstractFileSystem as input. To be used for testing.
343       */
344    
345      /**
346       * Create a FileContext with specified FS as default using the specified
347       * config.
348       * 
349       * @param defFS
350       * @param aConf
351       * @return new FileContext with specifed FS as default.
352       */
353      public static FileContext getFileContext(final AbstractFileSystem defFS,
354                        final Configuration aConf) {
355        return new FileContext(defFS, FsPermission.getUMask(aConf), aConf);
356      }
357      
358      /**
359       * Create a FileContext for specified file system using the default config.
360       * 
361       * @param defaultFS
362       * @return a FileContext with the specified AbstractFileSystem
363       *                 as the default FS.
364       */
365      protected static FileContext getFileContext(
366        final AbstractFileSystem defaultFS) {
367        return getFileContext(defaultFS, new Configuration());
368      }
369     
370      /**
371       * Static Factory methods for getting a FileContext.
372       * Note new file contexts are created for each call.
373       * The only singleton is the local FS context using the default config.
374       * 
375       * Methods that use the default config: the default config read from the
376       * $HADOOP_CONFIG/core.xml,
377       * Unspecified key-values for config are defaulted from core-defaults.xml
378       * in the release jar.
379       * 
380       * The keys relevant to the FileContext layer are extracted at time of
381       * construction. Changes to the config after the call are ignore
382       * by the FileContext layer. 
383       * The conf is passed to lower layers like AbstractFileSystem and HDFS which
384       * pick up their own config variables.
385       */
386    
387      /**
388       * Create a FileContext using the default config read from the
389       * $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
390       * from core-defaults.xml in the release jar.
391       * 
392       * @throws UnsupportedFileSystemException If the file system from the default
393       *           configuration is not supported
394       */
395      public static FileContext getFileContext()
396          throws UnsupportedFileSystemException {
397        return getFileContext(new Configuration());
398      }
399    
400      /**
401       * @return a FileContext for the local file system using the default config.
402       * @throws UnsupportedFileSystemException If the file system for
403       *           {@link FsConstants#LOCAL_FS_URI} is not supported.
404       */
405      public static FileContext getLocalFSFileContext()
406          throws UnsupportedFileSystemException {
407        return getFileContext(FsConstants.LOCAL_FS_URI);
408      }
409    
410      /**
411       * Create a FileContext for specified URI using the default config.
412       * 
413       * @param defaultFsUri
414       * @return a FileContext with the specified URI as the default FS.
415       * 
416       * @throws UnsupportedFileSystemException If the file system for
417       *           <code>defaultFsUri</code> is not supported
418       */
419      public static FileContext getFileContext(final URI defaultFsUri)
420          throws UnsupportedFileSystemException {
421        return getFileContext(defaultFsUri, new Configuration());
422      }
423    
424      /**
425       * Create a FileContext for specified default URI using the specified config.
426       * 
427       * @param defaultFsUri
428       * @param aConf
429       * @return new FileContext for specified uri
430       * @throws UnsupportedFileSystemException If the file system with specified is
431       *           not supported
432       * @throws RuntimeException If the file system specified is supported but
433       *         could not be instantiated, or if login fails.
434       */
435      public static FileContext getFileContext(final URI defaultFsUri,
436          final Configuration aConf) throws UnsupportedFileSystemException {
437        UserGroupInformation currentUser = null;
438        AbstractFileSystem defaultAfs = null;
439        try {
440          currentUser = UserGroupInformation.getCurrentUser();
441          defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf);
442        } catch (UnsupportedFileSystemException ex) {
443          throw ex;
444        } catch (IOException ex) {
445          LOG.error(ex);
446          throw new RuntimeException(ex);
447        }
448        return getFileContext(defaultAfs, aConf);
449      }
450    
451      /**
452       * Create a FileContext using the passed config. Generally it is better to use
453       * {@link #getFileContext(URI, Configuration)} instead of this one.
454       * 
455       * 
456       * @param aConf
457       * @return new FileContext
458       * @throws UnsupportedFileSystemException If file system in the config
459       *           is not supported
460       */
461      public static FileContext getFileContext(final Configuration aConf)
462          throws UnsupportedFileSystemException {
463        return getFileContext(
464          URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)), 
465          aConf);
466      }
467    
468      /**
469       * @param aConf - from which the FileContext is configured
470       * @return a FileContext for the local file system using the specified config.
471       * 
472       * @throws UnsupportedFileSystemException If default file system in the config
473       *           is not supported
474       * 
475       */
476      public static FileContext getLocalFSFileContext(final Configuration aConf)
477          throws UnsupportedFileSystemException {
478        return getFileContext(FsConstants.LOCAL_FS_URI, aConf);
479      }
480    
481      /* This method is needed for tests. */
482      @InterfaceAudience.Private
483      @InterfaceStability.Unstable /* return type will change to AFS once
484                                      HADOOP-6223 is completed */
485      public AbstractFileSystem getDefaultFileSystem() {
486        return defaultFS;
487      }
488      
489      /**
490       * Set the working directory for wd-relative names (such a "foo/bar"). Working
491       * directory feature is provided by simply prefixing relative names with the
492       * working dir. Note this is different from Unix where the wd is actually set
493       * to the inode. Hence setWorkingDir does not follow symlinks etc. This works
494       * better in a distributed environment that has multiple independent roots.
495       * {@link #getWorkingDirectory()} should return what setWorkingDir() set.
496       * 
497       * @param newWDir new working directory
498       * @throws IOException 
499       * <br>
500       *           NewWdir can be one of:
501       *           <ul>
502       *           <li>relative path: "foo/bar";</li>
503       *           <li>absolute without scheme: "/foo/bar"</li>
504       *           <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
505       *           </ul>
506       * <br>
507       *           Illegal WDs:
508       *           <ul>
509       *           <li>relative with scheme: "xx:foo/bar"</li>
510       *           <li>non existent directory</li>
511       *           </ul>
512       */
513      public void setWorkingDirectory(final Path newWDir) throws IOException {
514        checkNotSchemeWithRelative(newWDir);
515        /* wd is stored as a fully qualified path. We check if the given 
516         * path is not relative first since resolve requires and returns 
517         * an absolute path.
518         */  
519        final Path newWorkingDir = new Path(workingDir, newWDir);
520        FileStatus status = getFileStatus(newWorkingDir);
521        if (status.isFile()) {
522          throw new FileNotFoundException("Cannot setWD to a file");
523        }
524        workingDir = newWorkingDir;
525      }
526      
527      /**
528       * Gets the working directory for wd-relative names (such a "foo/bar").
529       */
530      public Path getWorkingDirectory() {
531        return workingDir;
532      }
533      
534      /**
535       * Gets the ugi in the file-context
536       * @return UserGroupInformation
537       */
538      public UserGroupInformation getUgi() {
539        return ugi;
540      }
541      
542      /**
543       * Return the current user's home directory in this file system.
544       * The default implementation returns "/user/$USER/".
545       * @return the home directory
546       */
547      public Path getHomeDirectory() {
548        return defaultFS.getHomeDirectory();
549      }
550      
551      /**
552       * 
553       * @return the umask of this FileContext
554       */
555      public FsPermission getUMask() {
556        return umask;
557      }
558      
559      /**
560       * Set umask to the supplied parameter.
561       * @param newUmask  the new umask
562       */
563      public void setUMask(final FsPermission newUmask) {
564        umask = newUmask;
565      }
566      
567      
568      /**
569       * Resolve the path following any symlinks or mount points
570       * @param f to be resolved
571       * @return fully qualified resolved path
572       * 
573       * @throws FileNotFoundException  If <code>f</code> does not exist
574       * @throws AccessControlException if access denied
575       * @throws IOException If an IO Error occurred
576       * 
577       * Exceptions applicable to file systems accessed over RPC:
578       * @throws RpcClientException If an exception occurred in the RPC client
579       * @throws RpcServerException If an exception occurred in the RPC server
580       * @throws UnexpectedServerException If server implementation throws
581       *           undeclared exception to RPC server
582       * 
583       * RuntimeExceptions:
584       * @throws InvalidPathException If path <code>f</code> is not valid
585       */
586      public Path resolvePath(final Path f) throws FileNotFoundException,
587          UnresolvedLinkException, AccessControlException, IOException {
588        return resolve(f);
589      }
590      
591      /**
592       * Make the path fully qualified if it is isn't. 
593       * A Fully-qualified path has scheme and authority specified and an absolute
594       * path.
595       * Use the default file system and working dir in this FileContext to qualify.
596       * @param path
597       * @return qualified path
598       */
599      public Path makeQualified(final Path path) {
600        return path.makeQualified(defaultFS.getUri(), getWorkingDirectory());
601      }
602    
603      /**
604       * Create or overwrite file on indicated path and returns an output stream for
605       * writing into the file.
606       * 
607       * @param f the file name to open
608       * @param createFlag gives the semantics of create; see {@link CreateFlag}
609       * @param opts file creation options; see {@link Options.CreateOpts}.
610       *          <ul>
611       *          <li>Progress - to report progress on the operation - default null
612       *          <li>Permission - umask is applied against permisssion: default is
613       *          FsPermissions:getDefault()
614       * 
615       *          <li>CreateParent - create missing parent path; default is to not
616       *          to create parents
617       *          <li>The defaults for the following are SS defaults of the file
618       *          server implementing the target path. Not all parameters make sense
619       *          for all kinds of file system - eg. localFS ignores Blocksize,
620       *          replication, checksum
621       *          <ul>
622       *          <li>BufferSize - buffersize used in FSDataOutputStream
623       *          <li>Blocksize - block size for file blocks
624       *          <li>ReplicationFactor - replication for blocks
625       *          <li>ChecksumParam - Checksum parameters. server default is used
626       *          if not specified.
627       *          </ul>
628       *          </ul>
629       * 
630       * @return {@link FSDataOutputStream} for created file
631       * 
632       * @throws AccessControlException If access is denied
633       * @throws FileAlreadyExistsException If file <code>f</code> already exists
634       * @throws FileNotFoundException If parent of <code>f</code> does not exist
635       *           and <code>createParent</code> is false
636       * @throws ParentNotDirectoryException If parent of <code>f</code> is not a
637       *           directory.
638       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
639       *           not supported
640       * @throws IOException If an I/O error occurred
641       * 
642       * Exceptions applicable to file systems accessed over RPC:
643       * @throws RpcClientException If an exception occurred in the RPC client
644       * @throws RpcServerException If an exception occurred in the RPC server
645       * @throws UnexpectedServerException If server implementation throws
646       *           undeclared exception to RPC server
647       * 
648       * RuntimeExceptions:
649       * @throws InvalidPathException If path <code>f</code> is not valid
650       */
651      public FSDataOutputStream create(final Path f,
652          final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
653          throws AccessControlException, FileAlreadyExistsException,
654          FileNotFoundException, ParentNotDirectoryException,
655          UnsupportedFileSystemException, IOException {
656        Path absF = fixRelativePart(f);
657    
658        // If one of the options is a permission, extract it & apply umask
659        // If not, add a default Perms and apply umask;
660        // AbstractFileSystem#create
661    
662        CreateOpts.Perms permOpt = 
663          (CreateOpts.Perms) CreateOpts.getOpt(CreateOpts.Perms.class, opts);
664        FsPermission permission = (permOpt != null) ? permOpt.getValue() :
665                                          FILE_DEFAULT_PERM;
666        permission = permission.applyUMask(umask);
667    
668        final CreateOpts[] updatedOpts = 
669                          CreateOpts.setOpt(CreateOpts.perms(permission), opts);
670        return new FSLinkResolver<FSDataOutputStream>() {
671          public FSDataOutputStream next(final AbstractFileSystem fs, final Path p) 
672            throws IOException {
673            return fs.create(p, createFlag, updatedOpts);
674          }
675        }.resolve(this, absF);
676      }
677    
678      /**
679       * Make(create) a directory and all the non-existent parents.
680       * 
681       * @param dir - the dir to make
682       * @param permission - permissions is set permission&~umask
683       * @param createParent - if true then missing parent dirs are created if false
684       *          then parent must exist
685       * 
686       * @throws AccessControlException If access is denied
687       * @throws FileAlreadyExistsException If directory <code>dir</code> already
688       *           exists
689       * @throws FileNotFoundException If parent of <code>dir</code> does not exist
690       *           and <code>createParent</code> is false
691       * @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
692       *           directory
693       * @throws UnsupportedFileSystemException If file system for <code>dir</code>
694       *         is not supported
695       * @throws IOException If an I/O error occurred
696       * 
697       * Exceptions applicable to file systems accessed over RPC:
698       * @throws RpcClientException If an exception occurred in the RPC client
699       * @throws UnexpectedServerException If server implementation throws 
700       *           undeclared exception to RPC server
701       * 
702       * RuntimeExceptions:
703       * @throws InvalidPathException If path <code>dir</code> is not valid
704       */
705      public void mkdir(final Path dir, final FsPermission permission,
706          final boolean createParent) throws AccessControlException,
707          FileAlreadyExistsException, FileNotFoundException,
708          ParentNotDirectoryException, UnsupportedFileSystemException, 
709          IOException {
710        final Path absDir = fixRelativePart(dir);
711        final FsPermission absFerms = (permission == null ? 
712              FsPermission.getDirDefault() : permission).applyUMask(umask);
713        new FSLinkResolver<Void>() {
714          public Void next(final AbstractFileSystem fs, final Path p) 
715            throws IOException, UnresolvedLinkException {
716            fs.mkdir(p, absFerms, createParent);
717            return null;
718          }
719        }.resolve(this, absDir);
720      }
721    
722      /**
723       * Delete a file.
724       * @param f the path to delete.
725       * @param recursive if path is a directory and set to 
726       * true, the directory is deleted else throws an exception. In
727       * case of a file the recursive can be set to either true or false.
728       *
729       * @throws AccessControlException If access is denied
730       * @throws FileNotFoundException If <code>f</code> does not exist
731       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
732       *           not supported
733       * @throws IOException If an I/O error occurred
734       * 
735       * Exceptions applicable to file systems accessed over RPC:
736       * @throws RpcClientException If an exception occurred in the RPC client
737       * @throws RpcServerException If an exception occurred in the RPC server
738       * @throws UnexpectedServerException If server implementation throws 
739       *           undeclared exception to RPC server
740       * 
741       * RuntimeExceptions:
742       * @throws InvalidPathException If path <code>f</code> is invalid
743       */
744      public boolean delete(final Path f, final boolean recursive)
745          throws AccessControlException, FileNotFoundException,
746          UnsupportedFileSystemException, IOException {
747        Path absF = fixRelativePart(f);
748        return new FSLinkResolver<Boolean>() {
749          public Boolean next(final AbstractFileSystem fs, final Path p) 
750            throws IOException, UnresolvedLinkException {
751            return Boolean.valueOf(fs.delete(p, recursive));
752          }
753        }.resolve(this, absF);
754      }
755     
756      /**
757       * Opens an FSDataInputStream at the indicated Path using
758       * default buffersize.
759       * @param f the file name to open
760       *
761       * @throws AccessControlException If access is denied
762       * @throws FileNotFoundException If file <code>f</code> does not exist
763       * @throws UnsupportedFileSystemException If file system for <code>f</code>
764       *         is not supported
765       * @throws IOException If an I/O error occurred
766       * 
767       * Exceptions applicable to file systems accessed over RPC:
768       * @throws RpcClientException If an exception occurred in the RPC client
769       * @throws RpcServerException If an exception occurred in the RPC server
770       * @throws UnexpectedServerException If server implementation throws 
771       *           undeclared exception to RPC server
772       */
773      public FSDataInputStream open(final Path f) throws AccessControlException,
774          FileNotFoundException, UnsupportedFileSystemException, IOException {
775        final Path absF = fixRelativePart(f);
776        return new FSLinkResolver<FSDataInputStream>() {
777          public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 
778            throws IOException, UnresolvedLinkException {
779            return fs.open(p);
780          }
781        }.resolve(this, absF);
782      }
783    
784      /**
785       * Opens an FSDataInputStream at the indicated Path.
786       * 
787       * @param f the file name to open
788       * @param bufferSize the size of the buffer to be used.
789       * 
790       * @throws AccessControlException If access is denied
791       * @throws FileNotFoundException If file <code>f</code> does not exist
792       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
793       *           not supported
794       * @throws IOException If an I/O error occurred
795       * 
796       * Exceptions applicable to file systems accessed over RPC:
797       * @throws RpcClientException If an exception occurred in the RPC client
798       * @throws RpcServerException If an exception occurred in the RPC server
799       * @throws UnexpectedServerException If server implementation throws 
800       *           undeclared exception to RPC server
801       */
802      public FSDataInputStream open(final Path f, final int bufferSize)
803          throws AccessControlException, FileNotFoundException,
804          UnsupportedFileSystemException, IOException {
805        final Path absF = fixRelativePart(f);
806        return new FSLinkResolver<FSDataInputStream>() {
807          public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 
808            throws IOException, UnresolvedLinkException {
809            return fs.open(p, bufferSize);
810          }
811        }.resolve(this, absF);
812      }
813    
814      /**
815       * Set replication for an existing file.
816       * 
817       * @param f file name
818       * @param replication new replication
819       *
820       * @return true if successful
821       *
822       * @throws AccessControlException If access is denied
823       * @throws FileNotFoundException If file <code>f</code> does not exist
824       * @throws IOException If an I/O error occurred
825       * 
826       * Exceptions applicable to file systems accessed over RPC:
827       * @throws RpcClientException If an exception occurred in the RPC client
828       * @throws RpcServerException If an exception occurred in the RPC server
829       * @throws UnexpectedServerException If server implementation throws 
830       *           undeclared exception to RPC server
831       */
832      public boolean setReplication(final Path f, final short replication)
833          throws AccessControlException, FileNotFoundException,
834          IOException {
835        final Path absF = fixRelativePart(f);
836        return new FSLinkResolver<Boolean>() {
837          public Boolean next(final AbstractFileSystem fs, final Path p) 
838            throws IOException, UnresolvedLinkException {
839            return Boolean.valueOf(fs.setReplication(p, replication));
840          }
841        }.resolve(this, absF);
842      }
843    
844      /**
845       * Renames Path src to Path dst
846       * <ul>
847       * <li
848       * <li>Fails if src is a file and dst is a directory.
849       * <li>Fails if src is a directory and dst is a file.
850       * <li>Fails if the parent of dst does not exist or is a file.
851       * </ul>
852       * <p>
853       * If OVERWRITE option is not passed as an argument, rename fails if the dst
854       * already exists.
855       * <p>
856       * If OVERWRITE option is passed as an argument, rename overwrites the dst if
857       * it is a file or an empty directory. Rename fails if dst is a non-empty
858       * directory.
859       * <p>
860       * Note that atomicity of rename is dependent on the file system
861       * implementation. Please refer to the file system documentation for details
862       * <p>
863       * 
864       * @param src path to be renamed
865       * @param dst new path after rename
866       * 
867       * @throws AccessControlException If access is denied
868       * @throws FileAlreadyExistsException If <code>dst</code> already exists and
869       *           <code>options</options> has {@link Options.Rename#OVERWRITE} 
870       *           option false.
871       * @throws FileNotFoundException If <code>src</code> does not exist
872       * @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
873       *           directory
874       * @throws UnsupportedFileSystemException If file system for <code>src</code>
875       *           and <code>dst</code> is not supported
876       * @throws IOException If an I/O error occurred
877       * 
878       * Exceptions applicable to file systems accessed over RPC:
879       * @throws RpcClientException If an exception occurred in the RPC client
880       * @throws RpcServerException If an exception occurred in the RPC server
881       * @throws UnexpectedServerException If server implementation throws
882       *           undeclared exception to RPC server
883       */
884      public void rename(final Path src, final Path dst,
885          final Options.Rename... options) throws AccessControlException,
886          FileAlreadyExistsException, FileNotFoundException,
887          ParentNotDirectoryException, UnsupportedFileSystemException,
888          IOException {
889        final Path absSrc = fixRelativePart(src);
890        final Path absDst = fixRelativePart(dst);
891        AbstractFileSystem srcFS = getFSofPath(absSrc);
892        AbstractFileSystem dstFS = getFSofPath(absDst);
893        if(!srcFS.getUri().equals(dstFS.getUri())) {
894          throw new IOException("Renames across AbstractFileSystems not supported");
895        }
896        try {
897          srcFS.rename(absSrc, absDst, options);
898        } catch (UnresolvedLinkException e) {
899          /* We do not know whether the source or the destination path
900           * was unresolved. Resolve the source path up until the final
901           * path component, then fully resolve the destination. 
902           */
903          final Path source = resolveIntermediate(absSrc);    
904          new FSLinkResolver<Void>() {
905            public Void next(final AbstractFileSystem fs, final Path p) 
906              throws IOException, UnresolvedLinkException {
907              fs.rename(source, p, options);
908              return null;
909            }
910          }.resolve(this, absDst);
911        }
912      }
913      
914      /**
915       * Set permission of a path.
916       * @param f
917       * @param permission - the new absolute permission (umask is not applied)
918       *
919       * @throws AccessControlException If access is denied
920       * @throws FileNotFoundException If <code>f</code> does not exist
921       * @throws UnsupportedFileSystemException If file system for <code>f</code>
922       *         is not supported
923       * @throws IOException If an I/O error occurred
924       * 
925       * Exceptions applicable to file systems accessed over RPC:
926       * @throws RpcClientException If an exception occurred in the RPC client
927       * @throws RpcServerException If an exception occurred in the RPC server
928       * @throws UnexpectedServerException If server implementation throws 
929       *           undeclared exception to RPC server
930       */
931      public void setPermission(final Path f, final FsPermission permission)
932          throws AccessControlException, FileNotFoundException,
933          UnsupportedFileSystemException, IOException {
934        final Path absF = fixRelativePart(f);
935        new FSLinkResolver<Void>() {
936          public Void next(final AbstractFileSystem fs, final Path p) 
937            throws IOException, UnresolvedLinkException {
938            fs.setPermission(p, permission);
939            return null;
940          }
941        }.resolve(this, absF);
942      }
943    
944      /**
945       * Set owner of a path (i.e. a file or a directory). The parameters username
946       * and groupname cannot both be null.
947       * 
948       * @param f The path
949       * @param username If it is null, the original username remains unchanged.
950       * @param groupname If it is null, the original groupname remains unchanged.
951       * 
952       * @throws AccessControlException If access is denied
953       * @throws FileNotFoundException If <code>f</code> does not exist
954       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
955       *           not supported
956       * @throws IOException If an I/O error occurred
957       * 
958       * Exceptions applicable to file systems accessed over RPC:
959       * @throws RpcClientException If an exception occurred in the RPC client
960       * @throws RpcServerException If an exception occurred in the RPC server
961       * @throws UnexpectedServerException If server implementation throws 
962       *           undeclared exception to RPC server
963       * 
964       * RuntimeExceptions:
965       * @throws HadoopIllegalArgumentException If <code>username</code> or
966       *           <code>groupname</code> is invalid.
967       */
968      public void setOwner(final Path f, final String username,
969          final String groupname) throws AccessControlException,
970          UnsupportedFileSystemException, FileNotFoundException,
971          IOException {
972        if ((username == null) && (groupname == null)) {
973          throw new HadoopIllegalArgumentException(
974              "username and groupname cannot both be null");
975        }
976        final Path absF = fixRelativePart(f);
977        new FSLinkResolver<Void>() {
978          public Void next(final AbstractFileSystem fs, final Path p) 
979            throws IOException, UnresolvedLinkException {
980            fs.setOwner(p, username, groupname);
981            return null;
982          }
983        }.resolve(this, absF);
984      }
985    
986      /**
987       * Set access time of a file.
988       * @param f The path
989       * @param mtime Set the modification time of this file.
990       *        The number of milliseconds since epoch (Jan 1, 1970). 
991       *        A value of -1 means that this call should not set modification time.
992       * @param atime Set the access time of this file.
993       *        The number of milliseconds since Jan 1, 1970. 
994       *        A value of -1 means that this call should not set access time.
995       *
996       * @throws AccessControlException If access is denied
997       * @throws FileNotFoundException If <code>f</code> does not exist
998       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
999       *           not supported
1000       * @throws IOException If an I/O error occurred
1001       * 
1002       * Exceptions applicable to file systems accessed over RPC:
1003       * @throws RpcClientException If an exception occurred in the RPC client
1004       * @throws RpcServerException If an exception occurred in the RPC server
1005       * @throws UnexpectedServerException If server implementation throws 
1006       *           undeclared exception to RPC server
1007       */
1008      public void setTimes(final Path f, final long mtime, final long atime)
1009          throws AccessControlException, FileNotFoundException,
1010          UnsupportedFileSystemException, IOException {
1011        final Path absF = fixRelativePart(f);
1012        new FSLinkResolver<Void>() {
1013          public Void next(final AbstractFileSystem fs, final Path p) 
1014            throws IOException, UnresolvedLinkException {
1015            fs.setTimes(p, mtime, atime);
1016            return null;
1017          }
1018        }.resolve(this, absF);
1019      }
1020    
1021      /**
1022       * Get the checksum of a file.
1023       *
1024       * @param f file path
1025       *
1026       * @return The file checksum.  The default return value is null,
1027       *  which indicates that no checksum algorithm is implemented
1028       *  in the corresponding FileSystem.
1029       *
1030       * @throws AccessControlException If access is denied
1031       * @throws FileNotFoundException If <code>f</code> does not exist
1032       * @throws IOException If an I/O error occurred
1033       * 
1034       * Exceptions applicable to file systems accessed over RPC:
1035       * @throws RpcClientException If an exception occurred in the RPC client
1036       * @throws RpcServerException If an exception occurred in the RPC server
1037       * @throws UnexpectedServerException If server implementation throws 
1038       *           undeclared exception to RPC server
1039       */
1040      public FileChecksum getFileChecksum(final Path f)
1041          throws AccessControlException, FileNotFoundException,
1042          IOException {
1043        final Path absF = fixRelativePart(f);
1044        return new FSLinkResolver<FileChecksum>() {
1045          public FileChecksum next(final AbstractFileSystem fs, final Path p) 
1046            throws IOException, UnresolvedLinkException {
1047            return fs.getFileChecksum(p);
1048          }
1049        }.resolve(this, absF);
1050      }
1051    
1052      /**
1053       * Set the verify checksum flag for the  file system denoted by the path.
1054       * This is only applicable if the 
1055       * corresponding FileSystem supports checksum. By default doesn't do anything.
1056       * @param verifyChecksum
1057       * @param f set the verifyChecksum for the Filesystem containing this path
1058       *
1059       * @throws AccessControlException If access is denied
1060       * @throws FileNotFoundException If <code>f</code> does not exist
1061       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1062       *           not supported
1063       * @throws IOException If an I/O error occurred
1064       * 
1065       * Exceptions applicable to file systems accessed over RPC:
1066       * @throws RpcClientException If an exception occurred in the RPC client
1067       * @throws RpcServerException If an exception occurred in the RPC server
1068       * @throws UnexpectedServerException If server implementation throws 
1069       *           undeclared exception to RPC server
1070       */
1071      public void setVerifyChecksum(final boolean verifyChecksum, final Path f)
1072          throws AccessControlException, FileNotFoundException,
1073          UnsupportedFileSystemException, IOException {
1074        final Path absF = resolve(fixRelativePart(f));
1075        getFSofPath(absF).setVerifyChecksum(verifyChecksum);
1076      }
1077    
1078      /**
1079       * Return a file status object that represents the path.
1080       * @param f The path we want information from
1081       *
1082       * @return a FileStatus object
1083       *
1084       * @throws AccessControlException If access is denied
1085       * @throws FileNotFoundException If <code>f</code> does not exist
1086       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1087       *           not supported
1088       * @throws IOException If an I/O error occurred
1089       * 
1090       * Exceptions applicable to file systems accessed over RPC:
1091       * @throws RpcClientException If an exception occurred in the RPC client
1092       * @throws RpcServerException If an exception occurred in the RPC server
1093       * @throws UnexpectedServerException If server implementation throws 
1094       *           undeclared exception to RPC server
1095       */
1096      public FileStatus getFileStatus(final Path f) throws AccessControlException,
1097          FileNotFoundException, UnsupportedFileSystemException, IOException {
1098        final Path absF = fixRelativePart(f);
1099        return new FSLinkResolver<FileStatus>() {
1100          public FileStatus next(final AbstractFileSystem fs, final Path p) 
1101            throws IOException, UnresolvedLinkException {
1102            return fs.getFileStatus(p);
1103          }
1104        }.resolve(this, absF);
1105      }
1106    
1107      /**
1108       * Return a fully qualified version of the given symlink target if it
1109       * has no scheme and authority. Partially and fully qualified paths 
1110       * are returned unmodified.
1111       * @param pathFS The AbstractFileSystem of the path
1112       * @param pathWithLink Path that contains the symlink
1113       * @param target The symlink's absolute target
1114       * @return Fully qualified version of the target.
1115       */
1116      private Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
1117        Path pathWithLink, Path target) {
1118        // NB: makeQualified uses the target's scheme and authority, if
1119        // specified, and the scheme and authority of pathFS, if not.
1120        final String scheme = target.toUri().getScheme();
1121        final String auth   = target.toUri().getAuthority();
1122        return (scheme == null && auth == null)
1123          ? target.makeQualified(pathFS.getUri(), pathWithLink.getParent())
1124          : target;
1125      }
1126      
1127      /**
1128       * Return a file status object that represents the path. If the path 
1129       * refers to a symlink then the FileStatus of the symlink is returned.
1130       * The behavior is equivalent to #getFileStatus() if the underlying
1131       * file system does not support symbolic links.
1132       * @param  f The path we want information from.
1133       * @return A FileStatus object
1134       * 
1135       * @throws AccessControlException If access is denied
1136       * @throws FileNotFoundException If <code>f</code> does not exist
1137       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1138       *           not supported
1139       * @throws IOException If an I/O error occurred
1140       */
1141      public FileStatus getFileLinkStatus(final Path f)
1142          throws AccessControlException, FileNotFoundException,
1143          UnsupportedFileSystemException, IOException {
1144        final Path absF = fixRelativePart(f);
1145        return new FSLinkResolver<FileStatus>() {
1146          public FileStatus next(final AbstractFileSystem fs, final Path p) 
1147            throws IOException, UnresolvedLinkException {
1148            FileStatus fi = fs.getFileLinkStatus(p);
1149            if (fi.isSymlink()) {
1150              fi.setSymlink(qualifySymlinkTarget(fs, p, fi.getSymlink()));
1151            }
1152            return fi;
1153          }
1154        }.resolve(this, absF);
1155      }
1156      
1157      /**
1158       * Returns the target of the given symbolic link as it was specified
1159       * when the link was created.  Links in the path leading up to the
1160       * final path component are resolved transparently.
1161       *
1162       * @param f the path to return the target of
1163       * @return The un-interpreted target of the symbolic link.
1164       * 
1165       * @throws AccessControlException If access is denied
1166       * @throws FileNotFoundException If path <code>f</code> does not exist
1167       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1168       *           not supported
1169       * @throws IOException If the given path does not refer to a symlink
1170       *           or an I/O error occurred
1171       */
1172      public Path getLinkTarget(final Path f) throws AccessControlException,
1173          FileNotFoundException, UnsupportedFileSystemException, IOException {
1174        final Path absF = fixRelativePart(f);
1175        return new FSLinkResolver<Path>() {
1176          public Path next(final AbstractFileSystem fs, final Path p) 
1177            throws IOException, UnresolvedLinkException {
1178            FileStatus fi = fs.getFileLinkStatus(p);
1179            return fi.getSymlink();
1180          }
1181        }.resolve(this, absF);
1182      }
1183      
1184      /**
1185       * Return blockLocation of the given file for the given offset and len.
1186       *  For a nonexistent file or regions, null will be returned.
1187       *
1188       * This call is most helpful with DFS, where it returns 
1189       * hostnames of machines that contain the given file.
1190       * 
1191       * @param f - get blocklocations of this file
1192       * @param start position (byte offset)
1193       * @param len (in bytes)
1194       *
1195       * @return block locations for given file at specified offset of len
1196       *
1197       * @throws AccessControlException If access is denied
1198       * @throws FileNotFoundException If <code>f</code> does not exist
1199       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1200       *           not supported
1201       * @throws IOException If an I/O error occurred
1202       * 
1203       * Exceptions applicable to file systems accessed over RPC:
1204       * @throws RpcClientException If an exception occurred in the RPC client
1205       * @throws RpcServerException If an exception occurred in the RPC server
1206       * @throws UnexpectedServerException If server implementation throws 
1207       *           undeclared exception to RPC server
1208       * 
1209       * RuntimeExceptions:
1210       * @throws InvalidPathException If path <code>f</code> is invalid
1211       */
1212      @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
1213      @InterfaceStability.Evolving
1214      public BlockLocation[] getFileBlockLocations(final Path f, final long start,
1215          final long len) throws AccessControlException, FileNotFoundException,
1216          UnsupportedFileSystemException, IOException {
1217        final Path absF = fixRelativePart(f);
1218        return new FSLinkResolver<BlockLocation[]>() {
1219          public BlockLocation[] next(final AbstractFileSystem fs, final Path p) 
1220            throws IOException, UnresolvedLinkException {
1221            return fs.getFileBlockLocations(p, start, len);
1222          }
1223        }.resolve(this, absF);
1224      }
1225      
1226      /**
1227       * Returns a status object describing the use and capacity of the
1228       * file system denoted by the Parh argument p.
1229       * If the file system has multiple partitions, the
1230       * use and capacity of the partition pointed to by the specified
1231       * path is reflected.
1232       * 
1233       * @param f Path for which status should be obtained. null means the
1234       * root partition of the default file system. 
1235       *
1236       * @return a FsStatus object
1237       *
1238       * @throws AccessControlException If access is denied
1239       * @throws FileNotFoundException If <code>f</code> does not exist
1240       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1241       *           not supported
1242       * @throws IOException If an I/O error occurred
1243       * 
1244       * Exceptions applicable to file systems accessed over RPC:
1245       * @throws RpcClientException If an exception occurred in the RPC client
1246       * @throws RpcServerException If an exception occurred in the RPC server
1247       * @throws UnexpectedServerException If server implementation throws 
1248       *           undeclared exception to RPC server
1249       */
1250      public FsStatus getFsStatus(final Path f) throws AccessControlException,
1251          FileNotFoundException, UnsupportedFileSystemException, IOException {
1252        if (f == null) {
1253          return defaultFS.getFsStatus();
1254        }
1255        final Path absF = fixRelativePart(f);
1256        return new FSLinkResolver<FsStatus>() {
1257          public FsStatus next(final AbstractFileSystem fs, final Path p) 
1258            throws IOException, UnresolvedLinkException {
1259            return fs.getFsStatus(p);
1260          }
1261        }.resolve(this, absF);
1262      }
1263    
1264      /**
1265       * Creates a symbolic link to an existing file. An exception is thrown if 
1266       * the symlink exits, the user does not have permission to create symlink,
1267       * or the underlying file system does not support symlinks.
1268       * 
1269       * Symlink permissions are ignored, access to a symlink is determined by
1270       * the permissions of the symlink target.
1271       * 
1272       * Symlinks in paths leading up to the final path component are resolved 
1273       * transparently. If the final path component refers to a symlink some 
1274       * functions operate on the symlink itself, these are:
1275       * - delete(f) and deleteOnExit(f) - Deletes the symlink.
1276       * - rename(src, dst) - If src refers to a symlink, the symlink is 
1277       *   renamed. If dst refers to a symlink, the symlink is over-written.
1278       * - getLinkTarget(f) - Returns the target of the symlink. 
1279       * - getFileLinkStatus(f) - Returns a FileStatus object describing
1280       *   the symlink.
1281       * Some functions, create() and mkdir(), expect the final path component
1282       * does not exist. If they are given a path that refers to a symlink that 
1283       * does exist they behave as if the path referred to an existing file or 
1284       * directory. All other functions fully resolve, ie follow, the symlink. 
1285       * These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
1286       * setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
1287       * getFsStatus, getFileStatus, exists, and listStatus.
1288       * 
1289       * Symlink targets are stored as given to createSymlink, assuming the 
1290       * underlying file system is capable of storing a fully qualified URI.
1291       * Dangling symlinks are permitted. FileContext supports four types of 
1292       * symlink targets, and resolves them as follows
1293       * <pre>
1294       * Given a path referring to a symlink of form:
1295       * 
1296       *   <---X---> 
1297       *   fs://host/A/B/link 
1298       *   <-----Y----->
1299       * 
1300       * In this path X is the scheme and authority that identify the file system,
1301       * and Y is the path leading up to the final path component "link". If Y is
1302       * a symlink  itself then let Y' be the target of Y and X' be the scheme and
1303       * authority of Y'. Symlink targets may:
1304       * 
1305       * 1. Fully qualified URIs
1306       * 
1307       * fs://hostX/A/B/file  Resolved according to the target file system.
1308       * 
1309       * 2. Partially qualified URIs (eg scheme but no host)
1310       * 
1311       * fs:///A/B/file  Resolved according to the target file sytem. Eg resolving
1312       *                 a symlink to hdfs:///A results in an exception because
1313       *                 HDFS URIs must be fully qualified, while a symlink to 
1314       *                 file:///A will not since Hadoop's local file systems 
1315       *                 require partially qualified URIs.
1316       * 
1317       * 3. Relative paths
1318       * 
1319       * path  Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 
1320       *       is "../B/file" then [Y'][path] is hdfs://host/B/file
1321       * 
1322       * 4. Absolute paths
1323       * 
1324       * path  Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
1325       *       is "/file" then [X][path] is hdfs://host/file
1326       * </pre>
1327       * 
1328       * @param target the target of the symbolic link
1329       * @param link the path to be created that points to target
1330       * @param createParent if true then missing parent dirs are created if 
1331       *                     false then parent must exist
1332       *
1333       *
1334       * @throws AccessControlException If access is denied
1335       * @throws FileAlreadyExistsException If file <code>linkcode> already exists
1336       * @throws FileNotFoundException If <code>target</code> does not exist
1337       * @throws ParentNotDirectoryException If parent of <code>link</code> is not a
1338       *           directory.
1339       * @throws UnsupportedFileSystemException If file system for 
1340       *           <code>target</code> or <code>link</code> is not supported
1341       * @throws IOException If an I/O error occurred
1342       */
1343      public void createSymlink(final Path target, final Path link,
1344          final boolean createParent) throws AccessControlException,
1345          FileAlreadyExistsException, FileNotFoundException,
1346          ParentNotDirectoryException, UnsupportedFileSystemException, 
1347          IOException { 
1348        final Path nonRelLink = fixRelativePart(link);
1349        new FSLinkResolver<Void>() {
1350          public Void next(final AbstractFileSystem fs, final Path p) 
1351            throws IOException, UnresolvedLinkException {
1352            fs.createSymlink(target, p, createParent);
1353            return null;
1354          }
1355        }.resolve(this, nonRelLink);
1356      }
1357      
1358      /**
1359       * List the statuses of the files/directories in the given path if the path is
1360       * a directory.
1361       * 
1362       * @param f is the path
1363       *
1364       * @return an iterator that traverses statuses of the files/directories 
1365       *         in the given path
1366       *
1367       * @throws AccessControlException If access is denied
1368       * @throws FileNotFoundException If <code>f</code> does not exist
1369       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1370       *           not supported
1371       * @throws IOException If an I/O error occurred
1372       * 
1373       * Exceptions applicable to file systems accessed over RPC:
1374       * @throws RpcClientException If an exception occurred in the RPC client
1375       * @throws RpcServerException If an exception occurred in the RPC server
1376       * @throws UnexpectedServerException If server implementation throws 
1377       *           undeclared exception to RPC server
1378       */
1379      public RemoteIterator<FileStatus> listStatus(final Path f) throws
1380          AccessControlException, FileNotFoundException,
1381          UnsupportedFileSystemException, IOException {
1382        final Path absF = fixRelativePart(f);
1383        return new FSLinkResolver<RemoteIterator<FileStatus>>() {
1384          public RemoteIterator<FileStatus> next(
1385              final AbstractFileSystem fs, final Path p) 
1386            throws IOException, UnresolvedLinkException {
1387            return fs.listStatusIterator(p);
1388          }
1389        }.resolve(this, absF);
1390      }
1391    
1392      /**
1393       * @return an iterator over the corrupt files under the given path
1394       * (may contain duplicates if a file has more than one corrupt block)
1395       * @throws IOException
1396       */
1397      public RemoteIterator<Path> listCorruptFileBlocks(Path path)
1398        throws IOException {
1399        final Path absF = fixRelativePart(path);
1400        return new FSLinkResolver<RemoteIterator<Path>>() {
1401          @Override
1402          public RemoteIterator<Path> next(final AbstractFileSystem fs,
1403                                           final Path p) 
1404            throws IOException, UnresolvedLinkException {
1405            return fs.listCorruptFileBlocks(p);
1406          }
1407        }.resolve(this, absF);
1408      }
1409      
1410      /**
1411       * List the statuses of the files/directories in the given path if the path is
1412       * a directory. 
1413       * Return the file's status and block locations If the path is a file.
1414       * 
1415       * If a returned status is a file, it contains the file's block locations.
1416       * 
1417       * @param f is the path
1418       *
1419       * @return an iterator that traverses statuses of the files/directories 
1420       *         in the given path
1421       * If any IO exception (for example the input directory gets deleted while
1422       * listing is being executed), next() or hasNext() of the returned iterator
1423       * may throw a RuntimeException with the io exception as the cause.
1424       *
1425       * @throws AccessControlException If access is denied
1426       * @throws FileNotFoundException If <code>f</code> does not exist
1427       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1428       *           not supported
1429       * @throws IOException If an I/O error occurred
1430       * 
1431       * Exceptions applicable to file systems accessed over RPC:
1432       * @throws RpcClientException If an exception occurred in the RPC client
1433       * @throws RpcServerException If an exception occurred in the RPC server
1434       * @throws UnexpectedServerException If server implementation throws 
1435       *           undeclared exception to RPC server
1436       */
1437      public RemoteIterator<LocatedFileStatus> listLocatedStatus(
1438          final Path f) throws
1439          AccessControlException, FileNotFoundException,
1440          UnsupportedFileSystemException, IOException {
1441        final Path absF = fixRelativePart(f);
1442        return new FSLinkResolver<RemoteIterator<LocatedFileStatus>>() {
1443          public RemoteIterator<LocatedFileStatus> next(
1444              final AbstractFileSystem fs, final Path p) 
1445            throws IOException, UnresolvedLinkException {
1446            return fs.listLocatedStatus(p);
1447          }
1448        }.resolve(this, absF);
1449      }
1450    
1451      /**
1452       * Mark a path to be deleted on JVM shutdown.
1453       * 
1454       * @param f the existing path to delete.
1455       *
1456       * @return  true if deleteOnExit is successful, otherwise false.
1457       *
1458       * @throws AccessControlException If access is denied
1459       * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1460       *           not supported
1461       * @throws IOException If an I/O error occurred
1462       * 
1463       * Exceptions applicable to file systems accessed over RPC:
1464       * @throws RpcClientException If an exception occurred in the RPC client
1465       * @throws RpcServerException If an exception occurred in the RPC server
1466       * @throws UnexpectedServerException If server implementation throws 
1467       *           undeclared exception to RPC server
1468       */
1469      public boolean deleteOnExit(Path f) throws AccessControlException,
1470          IOException {
1471        if (!this.util().exists(f)) {
1472          return false;
1473        }
1474        synchronized (DELETE_ON_EXIT) {
1475          if (DELETE_ON_EXIT.isEmpty()) {
1476            ShutdownHookManager.get().addShutdownHook(FINALIZER, SHUTDOWN_HOOK_PRIORITY);
1477          }
1478          
1479          Set<Path> set = DELETE_ON_EXIT.get(this);
1480          if (set == null) {
1481            set = new TreeSet<Path>();
1482            DELETE_ON_EXIT.put(this, set);
1483          }
1484          set.add(f);
1485        }
1486        return true;
1487      }
1488      
1489      private final Util util;
1490      public Util util() {
1491        return util;
1492      }
1493      
1494      
1495      /**
1496       * Utility/library methods built over the basic FileContext methods.
1497       * Since this are library functions, the oprtation are not atomic
1498       * and some of them may partially complete if other threads are making
1499       * changes to the same part of the name space.
1500       */
1501      public class Util {
1502        /**
1503         * Does the file exist?
1504         * Note: Avoid using this method if you already have FileStatus in hand.
1505         * Instead reuse the FileStatus 
1506         * @param f the  file or dir to be checked
1507         *
1508         * @throws AccessControlException If access is denied
1509         * @throws IOException If an I/O error occurred
1510         * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1511         *           not supported
1512         * 
1513         * Exceptions applicable to file systems accessed over RPC:
1514         * @throws RpcClientException If an exception occurred in the RPC client
1515         * @throws RpcServerException If an exception occurred in the RPC server
1516         * @throws UnexpectedServerException If server implementation throws 
1517         *           undeclared exception to RPC server
1518         */
1519        public boolean exists(final Path f) throws AccessControlException,
1520          UnsupportedFileSystemException, IOException {
1521          try {
1522            FileStatus fs = FileContext.this.getFileStatus(f);
1523            assert fs != null;
1524            return true;
1525          } catch (FileNotFoundException e) {
1526            return false;
1527          }
1528        }
1529        
1530        /**
1531         * Return a list of file status objects that corresponds to supplied paths
1532         * excluding those non-existent paths.
1533         * 
1534         * @param paths list of paths we want information from
1535         *
1536         * @return a list of FileStatus objects
1537         *
1538         * @throws AccessControlException If access is denied
1539         * @throws IOException If an I/O error occurred
1540         * 
1541         * Exceptions applicable to file systems accessed over RPC:
1542         * @throws RpcClientException If an exception occurred in the RPC client
1543         * @throws RpcServerException If an exception occurred in the RPC server
1544         * @throws UnexpectedServerException If server implementation throws 
1545         *           undeclared exception to RPC server
1546         */
1547        private FileStatus[] getFileStatus(Path[] paths)
1548            throws AccessControlException, IOException {
1549          if (paths == null) {
1550            return null;
1551          }
1552          ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
1553          for (int i = 0; i < paths.length; i++) {
1554            try {
1555              results.add(FileContext.this.getFileStatus(paths[i]));
1556            } catch (FileNotFoundException fnfe) {
1557              // ignoring 
1558            }
1559          }
1560          return results.toArray(new FileStatus[results.size()]);
1561        }
1562        
1563        
1564        /**
1565         * Return the {@link ContentSummary} of path f.
1566         * @param f path
1567         *
1568         * @return the {@link ContentSummary} of path f.
1569         *
1570         * @throws AccessControlException If access is denied
1571         * @throws FileNotFoundException If <code>f</code> does not exist
1572         * @throws UnsupportedFileSystemException If file system for 
1573         *         <code>f</code> is not supported
1574         * @throws IOException If an I/O error occurred
1575         * 
1576         * Exceptions applicable to file systems accessed over RPC:
1577         * @throws RpcClientException If an exception occurred in the RPC client
1578         * @throws RpcServerException If an exception occurred in the RPC server
1579         * @throws UnexpectedServerException If server implementation throws 
1580         *           undeclared exception to RPC server
1581         */
1582        public ContentSummary getContentSummary(Path f)
1583            throws AccessControlException, FileNotFoundException,
1584            UnsupportedFileSystemException, IOException {
1585          FileStatus status = FileContext.this.getFileStatus(f);
1586          if (status.isFile()) {
1587            return new ContentSummary(status.getLen(), 1, 0);
1588          }
1589          long[] summary = {0, 0, 1};
1590          RemoteIterator<FileStatus> statusIterator = 
1591            FileContext.this.listStatus(f);
1592          while(statusIterator.hasNext()) {
1593            FileStatus s = statusIterator.next();
1594            ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) :
1595                                           new ContentSummary(s.getLen(), 1, 0);
1596            summary[0] += c.getLength();
1597            summary[1] += c.getFileCount();
1598            summary[2] += c.getDirectoryCount();
1599          }
1600          return new ContentSummary(summary[0], summary[1], summary[2]);
1601        }
1602        
1603        /**
1604         * See {@link #listStatus(Path[], PathFilter)}
1605         */
1606        public FileStatus[] listStatus(Path[] files) throws AccessControlException,
1607            FileNotFoundException, IOException {
1608          return listStatus(files, DEFAULT_FILTER);
1609        }
1610         
1611        /**
1612         * Filter files/directories in the given path using the user-supplied path
1613         * filter.
1614         * 
1615         * @param f is the path name
1616         * @param filter is the user-supplied path filter
1617         *
1618         * @return an array of FileStatus objects for the files under the given path
1619         *         after applying the filter
1620         *
1621         * @throws AccessControlException If access is denied
1622         * @throws FileNotFoundException If <code>f</code> does not exist
1623         * @throws UnsupportedFileSystemException If file system for 
1624         *         <code>pathPattern</code> is not supported
1625         * @throws IOException If an I/O error occurred
1626         * 
1627         * Exceptions applicable to file systems accessed over RPC:
1628         * @throws RpcClientException If an exception occurred in the RPC client
1629         * @throws RpcServerException If an exception occurred in the RPC server
1630         * @throws UnexpectedServerException If server implementation throws 
1631         *           undeclared exception to RPC server
1632         */
1633        public FileStatus[] listStatus(Path f, PathFilter filter)
1634            throws AccessControlException, FileNotFoundException,
1635            UnsupportedFileSystemException, IOException {
1636          ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1637          listStatus(results, f, filter);
1638          return results.toArray(new FileStatus[results.size()]);
1639        }
1640        
1641        /**
1642         * Filter files/directories in the given list of paths using user-supplied
1643         * path filter.
1644         * 
1645         * @param files is a list of paths
1646         * @param filter is the filter
1647         *
1648         * @return a list of statuses for the files under the given paths after
1649         *         applying the filter
1650         *
1651         * @throws AccessControlException If access is denied
1652         * @throws FileNotFoundException If a file in <code>files</code> does not 
1653         *           exist
1654         * @throws IOException If an I/O error occurred
1655         * 
1656         * Exceptions applicable to file systems accessed over RPC:
1657         * @throws RpcClientException If an exception occurred in the RPC client
1658         * @throws RpcServerException If an exception occurred in the RPC server
1659         * @throws UnexpectedServerException If server implementation throws 
1660         *           undeclared exception to RPC server
1661         */
1662        public FileStatus[] listStatus(Path[] files, PathFilter filter)
1663            throws AccessControlException, FileNotFoundException, IOException {
1664          ArrayList<FileStatus> results = new ArrayList<FileStatus>();
1665          for (int i = 0; i < files.length; i++) {
1666            listStatus(results, files[i], filter);
1667          }
1668          return results.toArray(new FileStatus[results.size()]);
1669        }
1670      
1671        /*
1672         * Filter files/directories in the given path using the user-supplied path
1673         * filter. Results are added to the given array <code>results</code>.
1674         */
1675        private void listStatus(ArrayList<FileStatus> results, Path f,
1676            PathFilter filter) throws AccessControlException,
1677            FileNotFoundException, IOException {
1678          FileStatus[] listing = listStatus(f);
1679          if (listing != null) {
1680            for (int i = 0; i < listing.length; i++) {
1681              if (filter.accept(listing[i].getPath())) {
1682                results.add(listing[i]);
1683              }
1684            }
1685          }
1686        }
1687    
1688        /**
1689         * List the statuses of the files/directories in the given path 
1690         * if the path is a directory.
1691         * 
1692         * @param f is the path
1693         *
1694         * @return an array that contains statuses of the files/directories 
1695         *         in the given path
1696         *
1697         * @throws AccessControlException If access is denied
1698         * @throws FileNotFoundException If <code>f</code> does not exist
1699         * @throws UnsupportedFileSystemException If file system for <code>f</code> is
1700         *           not supported
1701         * @throws IOException If an I/O error occurred
1702         * 
1703         * Exceptions applicable to file systems accessed over RPC:
1704         * @throws RpcClientException If an exception occurred in the RPC client
1705         * @throws RpcServerException If an exception occurred in the RPC server
1706         * @throws UnexpectedServerException If server implementation throws 
1707         *           undeclared exception to RPC server
1708         */
1709        public FileStatus[] listStatus(final Path f) throws AccessControlException,
1710            FileNotFoundException, UnsupportedFileSystemException,
1711            IOException {
1712          final Path absF = fixRelativePart(f);
1713          return new FSLinkResolver<FileStatus[]>() {
1714            public FileStatus[] next(final AbstractFileSystem fs, final Path p) 
1715              throws IOException, UnresolvedLinkException {
1716              return fs.listStatus(p);
1717            }
1718          }.resolve(FileContext.this, absF);
1719        }
1720    
1721        /**
1722         * List the statuses and block locations of the files in the given path.
1723         * 
1724         * If the path is a directory, 
1725         *   if recursive is false, returns files in the directory;
1726         *   if recursive is true, return files in the subtree rooted at the path.
1727         *   The subtree is traversed in the depth-first order.
1728         * If the path is a file, return the file's status and block locations.
1729         * Files across symbolic links are also returned.
1730         * 
1731         * @param f is the path
1732         * @param recursive if the subdirectories need to be traversed recursively
1733         *
1734         * @return an iterator that traverses statuses of the files
1735         * If any IO exception (for example a sub-directory gets deleted while
1736         * listing is being executed), next() or hasNext() of the returned iterator
1737         * may throw a RuntimeException with the IO exception as the cause.
1738         *
1739         * @throws AccessControlException If access is denied
1740         * @throws FileNotFoundException If <code>f</code> does not exist
1741         * @throws UnsupportedFileSystemException If file system for <code>f</code>
1742         *         is not supported
1743         * @throws IOException If an I/O error occurred
1744         * 
1745         * Exceptions applicable to file systems accessed over RPC:
1746         * @throws RpcClientException If an exception occurred in the RPC client
1747         * @throws RpcServerException If an exception occurred in the RPC server
1748         * @throws UnexpectedServerException If server implementation throws 
1749         *           undeclared exception to RPC server
1750         */
1751        public RemoteIterator<LocatedFileStatus> listFiles(
1752            final Path f, final boolean recursive) throws AccessControlException,
1753            FileNotFoundException, UnsupportedFileSystemException, 
1754            IOException {
1755          return new RemoteIterator<LocatedFileStatus>() {
1756            private Stack<RemoteIterator<LocatedFileStatus>> itors = 
1757              new Stack<RemoteIterator<LocatedFileStatus>>();
1758            RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f);
1759            LocatedFileStatus curFile;
1760    
1761            /**
1762             * Returns <tt>true</tt> if the iterator has more files.
1763             *
1764             * @return <tt>true</tt> if the iterator has more files.
1765             * @throws AccessControlException if not allowed to access next
1766             *                                file's status or locations
1767             * @throws FileNotFoundException if next file does not exist any more
1768             * @throws UnsupportedFileSystemException if next file's 
1769             *                                        fs is unsupported
1770             * @throws IOException for all other IO errors
1771             *                     for example, NameNode is not avaialbe or
1772             *                     NameNode throws IOException due to an error
1773             *                     while getting the status or block locations
1774             */
1775            @Override
1776            public boolean hasNext() throws IOException {
1777              while (curFile == null) {
1778                if (curItor.hasNext()) {
1779                  handleFileStat(curItor.next());
1780                } else if (!itors.empty()) {
1781                  curItor = itors.pop();
1782                } else {
1783                  return false;
1784                }
1785              }
1786              return true;
1787            }
1788    
1789            /**
1790             * Process the input stat.
1791             * If it is a file, return the file stat.
1792             * If it is a directory, traverse the directory if recursive is true;
1793             * ignore it if recursive is false.
1794             * If it is a symlink, resolve the symlink first and then process it
1795             * depending on if it is a file or directory.
1796             * @param stat input status
1797             * @throws AccessControlException if access is denied
1798             * @throws FileNotFoundException if file is not found
1799             * @throws UnsupportedFileSystemException if fs is not supported
1800             * @throws IOException for all other IO errors
1801             */
1802            private void handleFileStat(LocatedFileStatus stat)
1803            throws IOException {
1804              if (stat.isFile()) { // file
1805                curFile = stat;
1806              } else if (stat.isSymlink()) { // symbolic link
1807                // resolve symbolic link
1808                FileStatus symstat = FileContext.this.getFileStatus(
1809                    stat.getSymlink());
1810                if (symstat.isFile() || (recursive && symstat.isDirectory())) {
1811                  itors.push(curItor);
1812                  curItor = listLocatedStatus(stat.getPath());
1813                }
1814              } else if (recursive) { // directory
1815                itors.push(curItor);
1816                curItor = listLocatedStatus(stat.getPath());
1817              }
1818            }
1819    
1820            /**
1821             * Returns the next file's status with its block locations
1822             *
1823             * @throws AccessControlException if not allowed to access next
1824             *                                file's status or locations
1825             * @throws FileNotFoundException if next file does not exist any more
1826             * @throws UnsupportedFileSystemException if next file's 
1827             *                                        fs is unsupported
1828             * @throws IOException for all other IO errors
1829             *                     for example, NameNode is not avaialbe or
1830             *                     NameNode throws IOException due to an error
1831             *                     while getting the status or block locations
1832             */
1833            @Override
1834            public LocatedFileStatus next() throws IOException {
1835              if (hasNext()) {
1836                LocatedFileStatus result = curFile;
1837                curFile = null;
1838                return result;
1839              } 
1840              throw new java.util.NoSuchElementException("No more entry in " + f);
1841            }
1842          };
1843        }
1844    
1845        /**
1846         * <p>Return all the files that match filePattern and are not checksum
1847         * files. Results are sorted by their names.
1848         * 
1849         * <p>
1850         * A filename pattern is composed of <i>regular</i> characters and
1851         * <i>special pattern matching</i> characters, which are:
1852         *
1853         * <dl>
1854         *  <dd>
1855         *   <dl>
1856         *    <p>
1857         *    <dt> <tt> ? </tt>
1858         *    <dd> Matches any single character.
1859         *
1860         *    <p>
1861         *    <dt> <tt> * </tt>
1862         *    <dd> Matches zero or more characters.
1863         *
1864         *    <p>
1865         *    <dt> <tt> [<i>abc</i>] </tt>
1866         *    <dd> Matches a single character from character set
1867         *     <tt>{<i>a,b,c</i>}</tt>.
1868         *
1869         *    <p>
1870         *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
1871         *    <dd> Matches a single character from the character range
1872         *     <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be
1873         *     lexicographically less than or equal to character <tt><i>b</i></tt>.
1874         *
1875         *    <p>
1876         *    <dt> <tt> [^<i>a</i>] </tt>
1877         *    <dd> Matches a single char that is not from character set or range
1878         *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
1879         *     immediately to the right of the opening bracket.
1880         *
1881         *    <p>
1882         *    <dt> <tt> \<i>c</i> </tt>
1883         *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
1884         *
1885         *    <p>
1886         *    <dt> <tt> {ab,cd} </tt>
1887         *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
1888         *    
1889         *    <p>
1890         *    <dt> <tt> {ab,c{de,fh}} </tt>
1891         *    <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt>
1892         *
1893         *   </dl>
1894         *  </dd>
1895         * </dl>
1896         *
1897         * @param pathPattern a regular expression specifying a pth pattern
1898         *
1899         * @return an array of paths that match the path pattern
1900         *
1901         * @throws AccessControlException If access is denied
1902         * @throws UnsupportedFileSystemException If file system for 
1903         *         <code>pathPattern</code> is not supported
1904         * @throws IOException If an I/O error occurred
1905         * 
1906         * Exceptions applicable to file systems accessed over RPC:
1907         * @throws RpcClientException If an exception occurred in the RPC client
1908         * @throws RpcServerException If an exception occurred in the RPC server
1909         * @throws UnexpectedServerException If server implementation throws 
1910         *           undeclared exception to RPC server
1911         */
1912        public FileStatus[] globStatus(Path pathPattern)
1913            throws AccessControlException, UnsupportedFileSystemException,
1914            IOException {
1915          return globStatus(pathPattern, DEFAULT_FILTER);
1916        }
1917        
1918        /**
1919         * Return an array of FileStatus objects whose path names match pathPattern
1920         * and is accepted by the user-supplied path filter. Results are sorted by
1921         * their path names.
1922         * Return null if pathPattern has no glob and the path does not exist.
1923         * Return an empty array if pathPattern has a glob and no path matches it. 
1924         * 
1925         * @param pathPattern regular expression specifying the path pattern
1926         * @param filter user-supplied path filter
1927         *
1928         * @return an array of FileStatus objects
1929         *
1930         * @throws AccessControlException If access is denied
1931         * @throws UnsupportedFileSystemException If file system for 
1932         *         <code>pathPattern</code> is not supported
1933         * @throws IOException If an I/O error occurred
1934         * 
1935         * Exceptions applicable to file systems accessed over RPC:
1936         * @throws RpcClientException If an exception occurred in the RPC client
1937         * @throws RpcServerException If an exception occurred in the RPC server
1938         * @throws UnexpectedServerException If server implementation throws 
1939         *           undeclared exception to RPC server
1940         */
1941        public FileStatus[] globStatus(final Path pathPattern,
1942            final PathFilter filter) throws AccessControlException,
1943            UnsupportedFileSystemException, IOException {
1944          URI uri = getFSofPath(fixRelativePart(pathPattern)).getUri();
1945    
1946          String filename = pathPattern.toUri().getPath();
1947    
1948          List<String> filePatterns = GlobExpander.expand(filename);
1949          if (filePatterns.size() == 1) {
1950            Path absPathPattern = fixRelativePart(pathPattern);
1951            return globStatusInternal(uri, new Path(absPathPattern.toUri()
1952                .getPath()), filter);
1953          } else {
1954            List<FileStatus> results = new ArrayList<FileStatus>();
1955            for (String iFilePattern : filePatterns) {
1956              Path iAbsFilePattern = fixRelativePart(new Path(iFilePattern));
1957              FileStatus[] files = globStatusInternal(uri, iAbsFilePattern, filter);
1958              for (FileStatus file : files) {
1959                results.add(file);
1960              }
1961            }
1962            return results.toArray(new FileStatus[results.size()]);
1963          }
1964        }
1965    
1966        /**
1967         * 
1968         * @param uri for all the inPathPattern
1969         * @param inPathPattern - without the scheme & authority (take from uri)
1970         * @param filter
1971         *
1972         * @return an array of FileStatus objects
1973         *
1974         * @throws AccessControlException If access is denied
1975         * @throws IOException If an I/O error occurred
1976         */
1977        private FileStatus[] globStatusInternal(final URI uri,
1978            final Path inPathPattern, final PathFilter filter)
1979            throws AccessControlException, IOException
1980          {
1981          Path[] parents = new Path[1];
1982          int level = 0;
1983          
1984          assert(inPathPattern.toUri().getScheme() == null &&
1985              inPathPattern.toUri().getAuthority() == null && 
1986              inPathPattern.isUriPathAbsolute());
1987    
1988          
1989          String filename = inPathPattern.toUri().getPath();
1990          
1991          // path has only zero component
1992          if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
1993            Path p = inPathPattern.makeQualified(uri, null);
1994            return getFileStatus(new Path[]{p});
1995          }
1996    
1997          // path has at least one component
1998          String[] components = filename.split(Path.SEPARATOR);
1999          
2000          // Path is absolute, first component is "/" hence first component
2001          // is the uri root
2002          parents[0] = new Path(new Path(uri), new Path("/"));
2003          level = 1;
2004    
2005          // glob the paths that match the parent path, ie. [0, components.length-1]
2006          boolean[] hasGlob = new boolean[]{false};
2007          Path[] relParentPaths = 
2008            globPathsLevel(parents, components, level, hasGlob);
2009          FileStatus[] results;
2010          
2011          if (relParentPaths == null || relParentPaths.length == 0) {
2012            results = null;
2013          } else {
2014            // fix the pathes to be abs
2015            Path[] parentPaths = new Path [relParentPaths.length]; 
2016            for(int i=0; i<relParentPaths.length; i++) {
2017              parentPaths[i] = relParentPaths[i].makeQualified(uri, null);
2018            }
2019            
2020            // Now work on the last component of the path
2021            GlobFilter fp = 
2022                        new GlobFilter(components[components.length - 1], filter);
2023            if (fp.hasPattern()) { // last component has a pattern
2024              // list parent directories and then glob the results
2025              try {
2026                results = listStatus(parentPaths, fp);
2027              } catch (FileNotFoundException e) {
2028                results = null;
2029              }
2030              hasGlob[0] = true;
2031            } else { // last component does not have a pattern
2032              // get all the path names
2033              ArrayList<Path> filteredPaths = 
2034                                          new ArrayList<Path>(parentPaths.length);
2035              for (int i = 0; i < parentPaths.length; i++) {
2036                parentPaths[i] = new Path(parentPaths[i],
2037                  components[components.length - 1]);
2038                if (fp.accept(parentPaths[i])) {
2039                  filteredPaths.add(parentPaths[i]);
2040                }
2041              }
2042              // get all their statuses
2043              results = getFileStatus(
2044                  filteredPaths.toArray(new Path[filteredPaths.size()]));
2045            }
2046          }
2047    
2048          // Decide if the pathPattern contains a glob or not
2049          if (results == null) {
2050            if (hasGlob[0]) {
2051              results = new FileStatus[0];
2052            }
2053          } else {
2054            if (results.length == 0) {
2055              if (!hasGlob[0]) {
2056                results = null;
2057              }
2058            } else {
2059              Arrays.sort(results);
2060            }
2061          }
2062          return results;
2063        }
2064    
2065        /*
2066         * For a path of N components, return a list of paths that match the
2067         * components [<code>level</code>, <code>N-1</code>].
2068         */
2069        private Path[] globPathsLevel(Path[] parents, String[] filePattern,
2070            int level, boolean[] hasGlob) throws AccessControlException,
2071            FileNotFoundException, IOException {
2072          if (level == filePattern.length - 1) {
2073            return parents;
2074          }
2075          if (parents == null || parents.length == 0) {
2076            return null;
2077          }
2078          GlobFilter fp = new GlobFilter(filePattern[level]);
2079          if (fp.hasPattern()) {
2080            try {
2081              parents = FileUtil.stat2Paths(listStatus(parents, fp));
2082            } catch (FileNotFoundException e) {
2083              parents = null;
2084            }
2085            hasGlob[0] = true;
2086          } else {
2087            for (int i = 0; i < parents.length; i++) {
2088              parents[i] = new Path(parents[i], filePattern[level]);
2089            }
2090          }
2091          return globPathsLevel(parents, filePattern, level + 1, hasGlob);
2092        }
2093    
2094        /**
2095         * Copy file from src to dest. See
2096         * {@link #copy(Path, Path, boolean, boolean)}
2097         */
2098        public boolean copy(final Path src, final Path dst)
2099            throws AccessControlException, FileAlreadyExistsException,
2100            FileNotFoundException, ParentNotDirectoryException,
2101            UnsupportedFileSystemException, IOException {
2102          return copy(src, dst, false, false);
2103        }
2104        
2105        /**
2106         * Copy from src to dst, optionally deleting src and overwriting dst.
2107         * @param src
2108         * @param dst
2109         * @param deleteSource - delete src if true
2110         * @param overwrite  overwrite dst if true; throw IOException if dst exists
2111         *         and overwrite is false.
2112         *
2113         * @return true if copy is successful
2114         *
2115         * @throws AccessControlException If access is denied
2116         * @throws FileAlreadyExistsException If <code>dst</code> already exists
2117         * @throws FileNotFoundException If <code>src</code> does not exist
2118         * @throws ParentNotDirectoryException If parent of <code>dst</code> is not
2119         *           a directory
2120         * @throws UnsupportedFileSystemException If file system for 
2121         *         <code>src</code> or <code>dst</code> is not supported
2122         * @throws IOException If an I/O error occurred
2123         * 
2124         * Exceptions applicable to file systems accessed over RPC:
2125         * @throws RpcClientException If an exception occurred in the RPC client
2126         * @throws RpcServerException If an exception occurred in the RPC server
2127         * @throws UnexpectedServerException If server implementation throws 
2128         *           undeclared exception to RPC server
2129         * 
2130         * RuntimeExceptions:
2131         * @throws InvalidPathException If path <code>dst</code> is invalid
2132         */
2133        public boolean copy(final Path src, final Path dst, boolean deleteSource,
2134            boolean overwrite) throws AccessControlException,
2135            FileAlreadyExistsException, FileNotFoundException,
2136            ParentNotDirectoryException, UnsupportedFileSystemException, 
2137            IOException {
2138          checkNotSchemeWithRelative(src);
2139          checkNotSchemeWithRelative(dst);
2140          Path qSrc = makeQualified(src);
2141          Path qDst = makeQualified(dst);
2142          checkDest(qSrc.getName(), qDst, overwrite);
2143          FileStatus fs = FileContext.this.getFileStatus(qSrc);
2144          if (fs.isDirectory()) {
2145            checkDependencies(qSrc, qDst);
2146            mkdir(qDst, FsPermission.getDirDefault(), true);
2147            FileStatus[] contents = listStatus(qSrc);
2148            for (FileStatus content : contents) {
2149              copy(makeQualified(content.getPath()), makeQualified(new Path(qDst,
2150                  content.getPath().getName())), deleteSource, overwrite);
2151            }
2152          } else {
2153            InputStream in=null;
2154            OutputStream out = null;
2155            try {
2156              in = open(qSrc);
2157              EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of(
2158                  CreateFlag.CREATE, CreateFlag.OVERWRITE) : 
2159                    EnumSet.of(CreateFlag.CREATE);
2160              out = create(qDst, createFlag);
2161              IOUtils.copyBytes(in, out, conf, true);
2162            } catch (IOException e) {
2163              IOUtils.closeStream(out);
2164              IOUtils.closeStream(in);
2165              throw e;
2166            }
2167          }
2168          if (deleteSource) {
2169            return delete(qSrc, true);
2170          } else {
2171            return true;
2172          }
2173        }
2174      }
2175    
2176      /**
2177       * Check if copying srcName to dst would overwrite an existing 
2178       * file or directory.
2179       * @param srcName File or directory to be copied.
2180       * @param dst Destination to copy srcName to.
2181       * @param overwrite Whether it's ok to overwrite an existing file. 
2182       * @throws AccessControlException If access is denied.
2183       * @throws IOException If dst is an existing directory, or dst is an 
2184       * existing file and the overwrite option is not passed.
2185       */
2186      private void checkDest(String srcName, Path dst, boolean overwrite)
2187          throws AccessControlException, IOException {
2188        try {
2189          FileStatus dstFs = getFileStatus(dst);
2190          if (dstFs.isDirectory()) {
2191            if (null == srcName) {
2192              throw new IOException("Target " + dst + " is a directory");
2193            }
2194            // Recurse to check if dst/srcName exists.
2195            checkDest(null, new Path(dst, srcName), overwrite);
2196          } else if (!overwrite) {
2197            throw new IOException("Target " + new Path(dst, srcName)
2198                + " already exists");
2199          }
2200        } catch (FileNotFoundException e) {
2201          // dst does not exist - OK to copy.
2202        }
2203      }
2204       
2205      //
2206      // If the destination is a subdirectory of the source, then
2207      // generate exception
2208      //
2209      private static void checkDependencies(Path qualSrc, Path qualDst)
2210        throws IOException {
2211        if (isSameFS(qualSrc, qualDst)) {
2212          String srcq = qualSrc.toString() + Path.SEPARATOR;
2213          String dstq = qualDst.toString() + Path.SEPARATOR;
2214          if (dstq.startsWith(srcq)) {
2215            if (srcq.length() == dstq.length()) {
2216              throw new IOException("Cannot copy " + qualSrc + " to itself.");
2217            } else {
2218              throw new IOException("Cannot copy " + qualSrc +
2219                                 " to its subdirectory " + qualDst);
2220            }
2221          }
2222        }
2223      }
2224      
2225      /**
2226       * Are qualSrc and qualDst of the same file system?
2227       * @param qualPath1 - fully qualified path
2228       * @param qualPath2 - fully qualified path
2229       * @return
2230       */
2231      private static boolean isSameFS(Path qualPath1, Path qualPath2) {
2232        URI srcUri = qualPath1.toUri();
2233        URI dstUri = qualPath2.toUri();
2234        return (srcUri.getScheme().equals(dstUri.getScheme()) && 
2235            !(srcUri.getAuthority() != null && dstUri.getAuthority() != null && srcUri
2236            .getAuthority().equals(dstUri.getAuthority())));
2237      }
2238    
2239      /**
2240       * Deletes all the paths in deleteOnExit on JVM shutdown.
2241       */
2242      static class FileContextFinalizer implements Runnable {
2243        public synchronized void run() {
2244          processDeleteOnExit();
2245        }
2246      }
2247    
2248      /**
2249       * Resolves all symbolic links in the specified path.
2250       * Returns the new path object.
2251       */
2252      protected Path resolve(final Path f) throws FileNotFoundException,
2253          UnresolvedLinkException, AccessControlException, IOException {
2254        return new FSLinkResolver<Path>() {
2255          public Path next(final AbstractFileSystem fs, final Path p) 
2256            throws IOException, UnresolvedLinkException {
2257            return fs.resolvePath(p);
2258          }
2259        }.resolve(this, f);
2260      }
2261    
2262      /**
2263       * Resolves all symbolic links in the specified path leading up 
2264       * to, but not including the final path component.
2265       * @param f path to resolve
2266       * @return the new path object.
2267       */
2268      protected Path resolveIntermediate(final Path f) throws IOException {
2269        return new FSLinkResolver<FileStatus>() {
2270          public FileStatus next(final AbstractFileSystem fs, final Path p) 
2271            throws IOException, UnresolvedLinkException {
2272            return fs.getFileLinkStatus(p);
2273          }
2274        }.resolve(this, f).getPath();
2275      }
2276    
2277      /**
2278       * Returns the list of AbstractFileSystems accessed in the path. The list may
2279       * contain more than one AbstractFileSystems objects in case of symlinks.
2280       * 
2281       * @param f
2282       *          Path which needs to be resolved
2283       * @return List of AbstractFileSystems accessed in the path
2284       * @throws IOException
2285       */
2286      Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f)
2287          throws IOException {
2288        final Path absF = fixRelativePart(f);
2289        final HashSet<AbstractFileSystem> result 
2290          = new HashSet<AbstractFileSystem>();
2291        new FSLinkResolver<Void>() {
2292          public Void next(final AbstractFileSystem fs, final Path p)
2293              throws IOException, UnresolvedLinkException {
2294            result.add(fs);
2295            fs.getFileStatus(p);
2296            return null;
2297          }
2298        }.resolve(this, absF);
2299        return result;
2300      }
2301      
2302      /**
2303       * Class used to perform an operation on and resolve symlinks in a
2304       * path. The operation may potentially span multiple file systems.  
2305       */
2306      protected abstract class FSLinkResolver<T> {
2307        // The maximum number of symbolic link components in a path
2308        private static final int MAX_PATH_LINKS = 32;
2309    
2310        /**
2311         * Generic helper function overridden on instantiation to perform a 
2312         * specific operation on the given file system using the given path
2313         * which may result in an UnresolvedLinkException. 
2314         * @param fs AbstractFileSystem to perform the operation on.
2315         * @param p Path given the file system.
2316         * @return Generic type determined by the specific implementation.
2317         * @throws UnresolvedLinkException If symbolic link <code>path</code> could 
2318         *           not be resolved
2319         * @throws IOException an I/O error occured
2320         */
2321        public abstract T next(final AbstractFileSystem fs, final Path p) 
2322          throws IOException, UnresolvedLinkException;  
2323            
2324        /**
2325         * Performs the operation specified by the next function, calling it
2326         * repeatedly until all symlinks in the given path are resolved.
2327         * @param fc FileContext used to access file systems.
2328         * @param p The path to resolve symlinks in.
2329         * @return Generic type determined by the implementation of next.
2330         * @throws IOException
2331         */
2332        public T resolve(final FileContext fc, Path p) throws IOException {
2333          int count = 0;
2334          T in = null;
2335          Path first = p;
2336          // NB: More than one AbstractFileSystem can match a scheme, eg 
2337          // "file" resolves to LocalFs but could have come by RawLocalFs.
2338          AbstractFileSystem fs = fc.getFSofPath(p);      
2339    
2340          // Loop until all symlinks are resolved or the limit is reached
2341          for (boolean isLink = true; isLink;) {
2342            try {
2343              in = next(fs, p);
2344              isLink = false;
2345            } catch (UnresolvedLinkException e) {
2346              if (count++ > MAX_PATH_LINKS) {
2347                throw new IOException("Possible cyclic loop while " +
2348                                      "following symbolic link " + first);
2349              }
2350              // Resolve the first unresolved path component
2351              p = qualifySymlinkTarget(fs, p, fs.getLinkTarget(p));
2352              fs = fc.getFSofPath(p);
2353            }
2354          }
2355          return in;
2356        }
2357      }
2358      
2359      /**
2360       * Get the statistics for a particular file system
2361       * 
2362       * @param uri
2363       *          the uri to lookup the statistics. Only scheme and authority part
2364       *          of the uri are used as the key to store and lookup.
2365       * @return a statistics object
2366       */
2367      public static Statistics getStatistics(URI uri) {
2368        return AbstractFileSystem.getStatistics(uri);
2369      }
2370    
2371      /**
2372       * Clears all the statistics stored in AbstractFileSystem, for all the file
2373       * systems.
2374       */
2375      public static void clearStatistics() {
2376        AbstractFileSystem.clearStatistics();
2377      }
2378    
2379      /**
2380       * Prints the statistics to standard output. File System is identified by the
2381       * scheme and authority.
2382       */
2383      public static void printStatistics() {
2384        AbstractFileSystem.printStatistics();
2385      }
2386    
2387      /**
2388       * @return Map of uri and statistics for each filesystem instantiated. The uri
2389       *         consists of scheme and authority for the filesystem.
2390       */
2391      public static Map<URI, Statistics> getAllStatistics() {
2392        return AbstractFileSystem.getAllStatistics();
2393      }
2394      
2395      /**
2396       * Get delegation tokens for the file systems accessed for a given
2397       * path.
2398       * @param p Path for which delegations tokens are requested.
2399       * @param renewer the account name that is allowed to renew the token.
2400       * @return List of delegation tokens.
2401       * @throws IOException
2402       */
2403      @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
2404      public List<Token<?>> getDelegationTokens(
2405          Path p, String renewer) throws IOException {
2406        Set<AbstractFileSystem> afsSet = resolveAbstractFileSystems(p);
2407        List<Token<?>> tokenList = 
2408            new ArrayList<Token<?>>();
2409        for (AbstractFileSystem afs : afsSet) {
2410          List<Token<?>> afsTokens = afs.getDelegationTokens(renewer);
2411          tokenList.addAll(afsTokens);
2412        }
2413        return tokenList;
2414      }
2415    }