001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs; 019 020 import java.io.FileNotFoundException; 021 import java.io.IOException; 022 import java.io.InputStream; 023 import java.io.OutputStream; 024 import java.net.URI; 025 import java.security.PrivilegedExceptionAction; 026 import java.util.ArrayList; 027 import java.util.Arrays; 028 import java.util.EnumSet; 029 import java.util.HashSet; 030 import java.util.IdentityHashMap; 031 import java.util.List; 032 import java.util.Map; 033 import java.util.Set; 034 import java.util.Stack; 035 import java.util.TreeSet; 036 import java.util.Map.Entry; 037 038 import org.apache.commons.logging.Log; 039 import org.apache.commons.logging.LogFactory; 040 import org.apache.hadoop.HadoopIllegalArgumentException; 041 import org.apache.hadoop.classification.InterfaceAudience; 042 import org.apache.hadoop.classification.InterfaceStability; 043 import org.apache.hadoop.conf.Configuration; 044 import org.apache.hadoop.fs.FileSystem.Statistics; 045 import org.apache.hadoop.fs.Options.CreateOpts; 046 import org.apache.hadoop.fs.permission.FsPermission; 047 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; 048 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; 049 import org.apache.hadoop.io.IOUtils; 050 import org.apache.hadoop.ipc.RpcClientException; 051 import org.apache.hadoop.ipc.RpcServerException; 052 import org.apache.hadoop.ipc.UnexpectedServerException; 053 import org.apache.hadoop.fs.InvalidPathException; 054 import org.apache.hadoop.security.AccessControlException; 055 import org.apache.hadoop.security.UserGroupInformation; 056 import org.apache.hadoop.security.token.Token; 057 import org.apache.hadoop.util.ShutdownHookManager; 058 059 /** 060 * The FileContext class provides an interface for users of the Hadoop 061 * file system. It exposes a number of file system operations, e.g. create, 062 * open, list. 063 * 064 * <h2>Path Names</h2> 065 * 066 * The Hadoop file system supports a URI namespace and URI names. This enables 067 * multiple types of file systems to be referenced using fully-qualified URIs. 068 * Two common Hadoop file system implementations are 069 * <ul> 070 * <li>the local file system: file:///path 071 * <li>the HDFS file system: hdfs://nnAddress:nnPort/path 072 * </ul> 073 * 074 * The Hadoop file system also supports additional naming schemes besides URIs. 075 * Hadoop has the concept of a <i>default file system</i>, which implies a 076 * default URI scheme and authority. This enables <i>slash-relative names</i> 077 * relative to the default FS, which are more convenient for users and 078 * application writers. The default FS is typically set by the user's 079 * environment, though it can also be manually specified. 080 * <p> 081 * 082 * Hadoop also supports <i>working-directory-relative</i> names, which are paths 083 * relative to the current working directory (similar to Unix). The working 084 * directory can be in a different file system than the default FS. 085 * <p> 086 * Thus, Hadoop path names can be specified as one of the following: 087 * <ul> 088 * <li>a fully-qualified URI: scheme://authority/path (e.g. 089 * hdfs://nnAddress:nnPort/foo/bar) 090 * <li>a slash-relative name: path relative to the default file system (e.g. 091 * /foo/bar) 092 * <li>a working-directory-relative name: path relative to the working dir (e.g. 093 * foo/bar) 094 * </ul> 095 * Relative paths with scheme (scheme:foo/bar) are illegal. 096 * 097 * <h2>Role of FileContext and Configuration Defaults</h2> 098 * 099 * The FileContext is the analogue of per-process file-related state in Unix. It 100 * contains two properties: 101 * 102 * <ul> 103 * <li>the default file system (for resolving slash-relative names) 104 * <li>the umask (for file permissions) 105 * </ul> 106 * In general, these properties are obtained from the default configuration file 107 * in the user's environment (see {@link Configuration}). 108 * 109 * Further file system properties are specified on the server-side. File system 110 * operations default to using these server-side defaults unless otherwise 111 * specified. 112 * <p> 113 * The file system related server-side defaults are: 114 * <ul> 115 * <li> the home directory (default is "/user/userName") 116 * <li> the initial wd (only for local fs) 117 * <li> replication factor 118 * <li> block size 119 * <li> buffer size 120 * <li> encryptDataTransfer 121 * <li> checksum option. (checksumType and bytesPerChecksum) 122 * </ul> 123 * 124 * <h2>Example Usage</h2> 125 * 126 * Example 1: use the default config read from the $HADOOP_CONFIG/core.xml. 127 * Unspecified values come from core-defaults.xml in the release jar. 128 * <ul> 129 * <li> myFContext = FileContext.getFileContext(); // uses the default config 130 * // which has your default FS 131 * <li> myFContext.create(path, ...); 132 * <li> myFContext.setWorkingDir(path); 133 * <li> myFContext.open (path, ...); 134 * <li>... 135 * </ul> 136 * Example 2: Get a FileContext with a specific URI as the default FS 137 * <ul> 138 * <li> myFContext = FileContext.getFileContext(URI); 139 * <li> myFContext.create(path, ...); 140 * <li>... 141 * </ul> 142 * Example 3: FileContext with local file system as the default 143 * <ul> 144 * <li> myFContext = FileContext.getLocalFSFileContext(); 145 * <li> myFContext.create(path, ...); 146 * <li> ... 147 * </ul> 148 * Example 4: Use a specific config, ignoring $HADOOP_CONFIG 149 * Generally you should not need use a config unless you are doing 150 * <ul> 151 * <li> configX = someConfigSomeOnePassedToYou; 152 * <li> myFContext = getFileContext(configX); // configX is not changed, 153 * // is passed down 154 * <li> myFContext.create(path, ...); 155 * <li>... 156 * </ul> 157 * 158 */ 159 160 @InterfaceAudience.Public 161 @InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */ 162 public final class FileContext { 163 164 public static final Log LOG = LogFactory.getLog(FileContext.class); 165 /** 166 * Default permission for directory and symlink 167 * In previous versions, this default permission was also used to 168 * create files, so files created end up with ugo+x permission. 169 * See HADOOP-9155 for detail. 170 * Two new constants are added to solve this, please use 171 * {@link FileContext#DIR_DEFAULT_PERM} for directory, and use 172 * {@link FileContext#FILE_DEFAULT_PERM} for file. 173 * This constant is kept for compatibility. 174 */ 175 public static final FsPermission DEFAULT_PERM = FsPermission.getDefault(); 176 /** 177 * Default permission for directory 178 */ 179 public static final FsPermission DIR_DEFAULT_PERM = FsPermission.getDirDefault(); 180 /** 181 * Default permission for file 182 */ 183 public static final FsPermission FILE_DEFAULT_PERM = FsPermission.getFileDefault(); 184 185 /** 186 * Priority of the FileContext shutdown hook. 187 */ 188 public static final int SHUTDOWN_HOOK_PRIORITY = 20; 189 190 /** 191 * List of files that should be deleted on JVM shutdown. 192 */ 193 static final Map<FileContext, Set<Path>> DELETE_ON_EXIT = 194 new IdentityHashMap<FileContext, Set<Path>>(); 195 196 /** JVM shutdown hook thread. */ 197 static final FileContextFinalizer FINALIZER = 198 new FileContextFinalizer(); 199 200 private static final PathFilter DEFAULT_FILTER = new PathFilter() { 201 public boolean accept(final Path file) { 202 return true; 203 } 204 }; 205 206 /** 207 * The FileContext is defined by. 208 * 1) defaultFS (slash) 209 * 2) wd 210 * 3) umask 211 */ 212 private final AbstractFileSystem defaultFS; //default FS for this FileContext. 213 private Path workingDir; // Fully qualified 214 private FsPermission umask; 215 private final Configuration conf; 216 private final UserGroupInformation ugi; 217 218 private FileContext(final AbstractFileSystem defFs, 219 final FsPermission theUmask, final Configuration aConf) { 220 defaultFS = defFs; 221 umask = FsPermission.getUMask(aConf); 222 conf = aConf; 223 try { 224 ugi = UserGroupInformation.getCurrentUser(); 225 } catch (IOException e) { 226 LOG.error("Exception in getCurrentUser: ",e); 227 throw new RuntimeException("Failed to get the current user " + 228 "while creating a FileContext", e); 229 } 230 /* 231 * Init the wd. 232 * WorkingDir is implemented at the FileContext layer 233 * NOT at the AbstractFileSystem layer. 234 * If the DefaultFS, such as localFilesystem has a notion of 235 * builtin WD, we use that as the initial WD. 236 * Otherwise the WD is initialized to the home directory. 237 */ 238 workingDir = defaultFS.getInitialWorkingDirectory(); 239 if (workingDir == null) { 240 workingDir = defaultFS.getHomeDirectory(); 241 } 242 util = new Util(); // for the inner class 243 } 244 245 /* 246 * Remove relative part - return "absolute": 247 * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar" 248 * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path 249 * ("/foo/bar") are returned unchanged. 250 * 251 * Applications that use FileContext should use #makeQualified() since 252 * they really want a fully qualified URI. 253 * Hence this method is not called makeAbsolute() and 254 * has been deliberately declared private. 255 */ 256 private Path fixRelativePart(Path p) { 257 if (p.isUriPathAbsolute()) { 258 return p; 259 } else { 260 return new Path(workingDir, p); 261 } 262 } 263 264 /** 265 * Delete all the paths that were marked as delete-on-exit. 266 */ 267 static void processDeleteOnExit() { 268 synchronized (DELETE_ON_EXIT) { 269 Set<Entry<FileContext, Set<Path>>> set = DELETE_ON_EXIT.entrySet(); 270 for (Entry<FileContext, Set<Path>> entry : set) { 271 FileContext fc = entry.getKey(); 272 Set<Path> paths = entry.getValue(); 273 for (Path path : paths) { 274 try { 275 fc.delete(path, true); 276 } catch (IOException e) { 277 LOG.warn("Ignoring failure to deleteOnExit for path " + path); 278 } 279 } 280 } 281 DELETE_ON_EXIT.clear(); 282 } 283 } 284 285 /** 286 * Pathnames with scheme and relative path are illegal. 287 * @param path to be checked 288 */ 289 private static void checkNotSchemeWithRelative(final Path path) { 290 if (path.toUri().isAbsolute() && !path.isUriPathAbsolute()) { 291 throw new HadoopIllegalArgumentException( 292 "Unsupported name: has scheme but relative path-part"); 293 } 294 } 295 296 /** 297 * Get the file system of supplied path. 298 * 299 * @param absOrFqPath - absolute or fully qualified path 300 * @return the file system of the path 301 * 302 * @throws UnsupportedFileSystemException If the file system for 303 * <code>absOrFqPath</code> is not supported. 304 * @throws IOExcepton If the file system for <code>absOrFqPath</code> could 305 * not be instantiated. 306 */ 307 private AbstractFileSystem getFSofPath(final Path absOrFqPath) 308 throws UnsupportedFileSystemException, IOException { 309 checkNotSchemeWithRelative(absOrFqPath); 310 if (!absOrFqPath.isAbsolute() && absOrFqPath.toUri().getScheme() == null) { 311 throw new HadoopIllegalArgumentException( 312 "FileContext Bug: path is relative"); 313 } 314 315 try { 316 // Is it the default FS for this FileContext? 317 defaultFS.checkPath(absOrFqPath); 318 return defaultFS; 319 } catch (Exception e) { // it is different FileSystem 320 return getAbstractFileSystem(ugi, absOrFqPath.toUri(), conf); 321 } 322 } 323 324 private static AbstractFileSystem getAbstractFileSystem( 325 UserGroupInformation user, final URI uri, final Configuration conf) 326 throws UnsupportedFileSystemException, IOException { 327 try { 328 return user.doAs(new PrivilegedExceptionAction<AbstractFileSystem>() { 329 public AbstractFileSystem run() throws UnsupportedFileSystemException { 330 return AbstractFileSystem.get(uri, conf); 331 } 332 }); 333 } catch (InterruptedException ex) { 334 LOG.error(ex); 335 throw new IOException("Failed to get the AbstractFileSystem for path: " 336 + uri, ex); 337 } 338 } 339 340 /** 341 * Protected Static Factory methods for getting a FileContexts 342 * that take a AbstractFileSystem as input. To be used for testing. 343 */ 344 345 /** 346 * Create a FileContext with specified FS as default using the specified 347 * config. 348 * 349 * @param defFS 350 * @param aConf 351 * @return new FileContext with specifed FS as default. 352 */ 353 public static FileContext getFileContext(final AbstractFileSystem defFS, 354 final Configuration aConf) { 355 return new FileContext(defFS, FsPermission.getUMask(aConf), aConf); 356 } 357 358 /** 359 * Create a FileContext for specified file system using the default config. 360 * 361 * @param defaultFS 362 * @return a FileContext with the specified AbstractFileSystem 363 * as the default FS. 364 */ 365 protected static FileContext getFileContext( 366 final AbstractFileSystem defaultFS) { 367 return getFileContext(defaultFS, new Configuration()); 368 } 369 370 /** 371 * Static Factory methods for getting a FileContext. 372 * Note new file contexts are created for each call. 373 * The only singleton is the local FS context using the default config. 374 * 375 * Methods that use the default config: the default config read from the 376 * $HADOOP_CONFIG/core.xml, 377 * Unspecified key-values for config are defaulted from core-defaults.xml 378 * in the release jar. 379 * 380 * The keys relevant to the FileContext layer are extracted at time of 381 * construction. Changes to the config after the call are ignore 382 * by the FileContext layer. 383 * The conf is passed to lower layers like AbstractFileSystem and HDFS which 384 * pick up their own config variables. 385 */ 386 387 /** 388 * Create a FileContext using the default config read from the 389 * $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted 390 * from core-defaults.xml in the release jar. 391 * 392 * @throws UnsupportedFileSystemException If the file system from the default 393 * configuration is not supported 394 */ 395 public static FileContext getFileContext() 396 throws UnsupportedFileSystemException { 397 return getFileContext(new Configuration()); 398 } 399 400 /** 401 * @return a FileContext for the local file system using the default config. 402 * @throws UnsupportedFileSystemException If the file system for 403 * {@link FsConstants#LOCAL_FS_URI} is not supported. 404 */ 405 public static FileContext getLocalFSFileContext() 406 throws UnsupportedFileSystemException { 407 return getFileContext(FsConstants.LOCAL_FS_URI); 408 } 409 410 /** 411 * Create a FileContext for specified URI using the default config. 412 * 413 * @param defaultFsUri 414 * @return a FileContext with the specified URI as the default FS. 415 * 416 * @throws UnsupportedFileSystemException If the file system for 417 * <code>defaultFsUri</code> is not supported 418 */ 419 public static FileContext getFileContext(final URI defaultFsUri) 420 throws UnsupportedFileSystemException { 421 return getFileContext(defaultFsUri, new Configuration()); 422 } 423 424 /** 425 * Create a FileContext for specified default URI using the specified config. 426 * 427 * @param defaultFsUri 428 * @param aConf 429 * @return new FileContext for specified uri 430 * @throws UnsupportedFileSystemException If the file system with specified is 431 * not supported 432 * @throws RuntimeException If the file system specified is supported but 433 * could not be instantiated, or if login fails. 434 */ 435 public static FileContext getFileContext(final URI defaultFsUri, 436 final Configuration aConf) throws UnsupportedFileSystemException { 437 UserGroupInformation currentUser = null; 438 AbstractFileSystem defaultAfs = null; 439 try { 440 currentUser = UserGroupInformation.getCurrentUser(); 441 defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf); 442 } catch (UnsupportedFileSystemException ex) { 443 throw ex; 444 } catch (IOException ex) { 445 LOG.error(ex); 446 throw new RuntimeException(ex); 447 } 448 return getFileContext(defaultAfs, aConf); 449 } 450 451 /** 452 * Create a FileContext using the passed config. Generally it is better to use 453 * {@link #getFileContext(URI, Configuration)} instead of this one. 454 * 455 * 456 * @param aConf 457 * @return new FileContext 458 * @throws UnsupportedFileSystemException If file system in the config 459 * is not supported 460 */ 461 public static FileContext getFileContext(final Configuration aConf) 462 throws UnsupportedFileSystemException { 463 return getFileContext( 464 URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)), 465 aConf); 466 } 467 468 /** 469 * @param aConf - from which the FileContext is configured 470 * @return a FileContext for the local file system using the specified config. 471 * 472 * @throws UnsupportedFileSystemException If default file system in the config 473 * is not supported 474 * 475 */ 476 public static FileContext getLocalFSFileContext(final Configuration aConf) 477 throws UnsupportedFileSystemException { 478 return getFileContext(FsConstants.LOCAL_FS_URI, aConf); 479 } 480 481 /* This method is needed for tests. */ 482 @InterfaceAudience.Private 483 @InterfaceStability.Unstable /* return type will change to AFS once 484 HADOOP-6223 is completed */ 485 public AbstractFileSystem getDefaultFileSystem() { 486 return defaultFS; 487 } 488 489 /** 490 * Set the working directory for wd-relative names (such a "foo/bar"). Working 491 * directory feature is provided by simply prefixing relative names with the 492 * working dir. Note this is different from Unix where the wd is actually set 493 * to the inode. Hence setWorkingDir does not follow symlinks etc. This works 494 * better in a distributed environment that has multiple independent roots. 495 * {@link #getWorkingDirectory()} should return what setWorkingDir() set. 496 * 497 * @param newWDir new working directory 498 * @throws IOException 499 * <br> 500 * NewWdir can be one of: 501 * <ul> 502 * <li>relative path: "foo/bar";</li> 503 * <li>absolute without scheme: "/foo/bar"</li> 504 * <li>fully qualified with scheme: "xx://auth/foo/bar"</li> 505 * </ul> 506 * <br> 507 * Illegal WDs: 508 * <ul> 509 * <li>relative with scheme: "xx:foo/bar"</li> 510 * <li>non existent directory</li> 511 * </ul> 512 */ 513 public void setWorkingDirectory(final Path newWDir) throws IOException { 514 checkNotSchemeWithRelative(newWDir); 515 /* wd is stored as a fully qualified path. We check if the given 516 * path is not relative first since resolve requires and returns 517 * an absolute path. 518 */ 519 final Path newWorkingDir = new Path(workingDir, newWDir); 520 FileStatus status = getFileStatus(newWorkingDir); 521 if (status.isFile()) { 522 throw new FileNotFoundException("Cannot setWD to a file"); 523 } 524 workingDir = newWorkingDir; 525 } 526 527 /** 528 * Gets the working directory for wd-relative names (such a "foo/bar"). 529 */ 530 public Path getWorkingDirectory() { 531 return workingDir; 532 } 533 534 /** 535 * Gets the ugi in the file-context 536 * @return UserGroupInformation 537 */ 538 public UserGroupInformation getUgi() { 539 return ugi; 540 } 541 542 /** 543 * Return the current user's home directory in this file system. 544 * The default implementation returns "/user/$USER/". 545 * @return the home directory 546 */ 547 public Path getHomeDirectory() { 548 return defaultFS.getHomeDirectory(); 549 } 550 551 /** 552 * 553 * @return the umask of this FileContext 554 */ 555 public FsPermission getUMask() { 556 return umask; 557 } 558 559 /** 560 * Set umask to the supplied parameter. 561 * @param newUmask the new umask 562 */ 563 public void setUMask(final FsPermission newUmask) { 564 umask = newUmask; 565 } 566 567 568 /** 569 * Resolve the path following any symlinks or mount points 570 * @param f to be resolved 571 * @return fully qualified resolved path 572 * 573 * @throws FileNotFoundException If <code>f</code> does not exist 574 * @throws AccessControlException if access denied 575 * @throws IOException If an IO Error occurred 576 * 577 * Exceptions applicable to file systems accessed over RPC: 578 * @throws RpcClientException If an exception occurred in the RPC client 579 * @throws RpcServerException If an exception occurred in the RPC server 580 * @throws UnexpectedServerException If server implementation throws 581 * undeclared exception to RPC server 582 * 583 * RuntimeExceptions: 584 * @throws InvalidPathException If path <code>f</code> is not valid 585 */ 586 public Path resolvePath(final Path f) throws FileNotFoundException, 587 UnresolvedLinkException, AccessControlException, IOException { 588 return resolve(f); 589 } 590 591 /** 592 * Make the path fully qualified if it is isn't. 593 * A Fully-qualified path has scheme and authority specified and an absolute 594 * path. 595 * Use the default file system and working dir in this FileContext to qualify. 596 * @param path 597 * @return qualified path 598 */ 599 public Path makeQualified(final Path path) { 600 return path.makeQualified(defaultFS.getUri(), getWorkingDirectory()); 601 } 602 603 /** 604 * Create or overwrite file on indicated path and returns an output stream for 605 * writing into the file. 606 * 607 * @param f the file name to open 608 * @param createFlag gives the semantics of create; see {@link CreateFlag} 609 * @param opts file creation options; see {@link Options.CreateOpts}. 610 * <ul> 611 * <li>Progress - to report progress on the operation - default null 612 * <li>Permission - umask is applied against permisssion: default is 613 * FsPermissions:getDefault() 614 * 615 * <li>CreateParent - create missing parent path; default is to not 616 * to create parents 617 * <li>The defaults for the following are SS defaults of the file 618 * server implementing the target path. Not all parameters make sense 619 * for all kinds of file system - eg. localFS ignores Blocksize, 620 * replication, checksum 621 * <ul> 622 * <li>BufferSize - buffersize used in FSDataOutputStream 623 * <li>Blocksize - block size for file blocks 624 * <li>ReplicationFactor - replication for blocks 625 * <li>ChecksumParam - Checksum parameters. server default is used 626 * if not specified. 627 * </ul> 628 * </ul> 629 * 630 * @return {@link FSDataOutputStream} for created file 631 * 632 * @throws AccessControlException If access is denied 633 * @throws FileAlreadyExistsException If file <code>f</code> already exists 634 * @throws FileNotFoundException If parent of <code>f</code> does not exist 635 * and <code>createParent</code> is false 636 * @throws ParentNotDirectoryException If parent of <code>f</code> is not a 637 * directory. 638 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 639 * not supported 640 * @throws IOException If an I/O error occurred 641 * 642 * Exceptions applicable to file systems accessed over RPC: 643 * @throws RpcClientException If an exception occurred in the RPC client 644 * @throws RpcServerException If an exception occurred in the RPC server 645 * @throws UnexpectedServerException If server implementation throws 646 * undeclared exception to RPC server 647 * 648 * RuntimeExceptions: 649 * @throws InvalidPathException If path <code>f</code> is not valid 650 */ 651 public FSDataOutputStream create(final Path f, 652 final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts) 653 throws AccessControlException, FileAlreadyExistsException, 654 FileNotFoundException, ParentNotDirectoryException, 655 UnsupportedFileSystemException, IOException { 656 Path absF = fixRelativePart(f); 657 658 // If one of the options is a permission, extract it & apply umask 659 // If not, add a default Perms and apply umask; 660 // AbstractFileSystem#create 661 662 CreateOpts.Perms permOpt = 663 (CreateOpts.Perms) CreateOpts.getOpt(CreateOpts.Perms.class, opts); 664 FsPermission permission = (permOpt != null) ? permOpt.getValue() : 665 FILE_DEFAULT_PERM; 666 permission = permission.applyUMask(umask); 667 668 final CreateOpts[] updatedOpts = 669 CreateOpts.setOpt(CreateOpts.perms(permission), opts); 670 return new FSLinkResolver<FSDataOutputStream>() { 671 public FSDataOutputStream next(final AbstractFileSystem fs, final Path p) 672 throws IOException { 673 return fs.create(p, createFlag, updatedOpts); 674 } 675 }.resolve(this, absF); 676 } 677 678 /** 679 * Make(create) a directory and all the non-existent parents. 680 * 681 * @param dir - the dir to make 682 * @param permission - permissions is set permission&~umask 683 * @param createParent - if true then missing parent dirs are created if false 684 * then parent must exist 685 * 686 * @throws AccessControlException If access is denied 687 * @throws FileAlreadyExistsException If directory <code>dir</code> already 688 * exists 689 * @throws FileNotFoundException If parent of <code>dir</code> does not exist 690 * and <code>createParent</code> is false 691 * @throws ParentNotDirectoryException If parent of <code>dir</code> is not a 692 * directory 693 * @throws UnsupportedFileSystemException If file system for <code>dir</code> 694 * is not supported 695 * @throws IOException If an I/O error occurred 696 * 697 * Exceptions applicable to file systems accessed over RPC: 698 * @throws RpcClientException If an exception occurred in the RPC client 699 * @throws UnexpectedServerException If server implementation throws 700 * undeclared exception to RPC server 701 * 702 * RuntimeExceptions: 703 * @throws InvalidPathException If path <code>dir</code> is not valid 704 */ 705 public void mkdir(final Path dir, final FsPermission permission, 706 final boolean createParent) throws AccessControlException, 707 FileAlreadyExistsException, FileNotFoundException, 708 ParentNotDirectoryException, UnsupportedFileSystemException, 709 IOException { 710 final Path absDir = fixRelativePart(dir); 711 final FsPermission absFerms = (permission == null ? 712 FsPermission.getDirDefault() : permission).applyUMask(umask); 713 new FSLinkResolver<Void>() { 714 public Void next(final AbstractFileSystem fs, final Path p) 715 throws IOException, UnresolvedLinkException { 716 fs.mkdir(p, absFerms, createParent); 717 return null; 718 } 719 }.resolve(this, absDir); 720 } 721 722 /** 723 * Delete a file. 724 * @param f the path to delete. 725 * @param recursive if path is a directory and set to 726 * true, the directory is deleted else throws an exception. In 727 * case of a file the recursive can be set to either true or false. 728 * 729 * @throws AccessControlException If access is denied 730 * @throws FileNotFoundException If <code>f</code> does not exist 731 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 732 * not supported 733 * @throws IOException If an I/O error occurred 734 * 735 * Exceptions applicable to file systems accessed over RPC: 736 * @throws RpcClientException If an exception occurred in the RPC client 737 * @throws RpcServerException If an exception occurred in the RPC server 738 * @throws UnexpectedServerException If server implementation throws 739 * undeclared exception to RPC server 740 * 741 * RuntimeExceptions: 742 * @throws InvalidPathException If path <code>f</code> is invalid 743 */ 744 public boolean delete(final Path f, final boolean recursive) 745 throws AccessControlException, FileNotFoundException, 746 UnsupportedFileSystemException, IOException { 747 Path absF = fixRelativePart(f); 748 return new FSLinkResolver<Boolean>() { 749 public Boolean next(final AbstractFileSystem fs, final Path p) 750 throws IOException, UnresolvedLinkException { 751 return Boolean.valueOf(fs.delete(p, recursive)); 752 } 753 }.resolve(this, absF); 754 } 755 756 /** 757 * Opens an FSDataInputStream at the indicated Path using 758 * default buffersize. 759 * @param f the file name to open 760 * 761 * @throws AccessControlException If access is denied 762 * @throws FileNotFoundException If file <code>f</code> does not exist 763 * @throws UnsupportedFileSystemException If file system for <code>f</code> 764 * is not supported 765 * @throws IOException If an I/O error occurred 766 * 767 * Exceptions applicable to file systems accessed over RPC: 768 * @throws RpcClientException If an exception occurred in the RPC client 769 * @throws RpcServerException If an exception occurred in the RPC server 770 * @throws UnexpectedServerException If server implementation throws 771 * undeclared exception to RPC server 772 */ 773 public FSDataInputStream open(final Path f) throws AccessControlException, 774 FileNotFoundException, UnsupportedFileSystemException, IOException { 775 final Path absF = fixRelativePart(f); 776 return new FSLinkResolver<FSDataInputStream>() { 777 public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 778 throws IOException, UnresolvedLinkException { 779 return fs.open(p); 780 } 781 }.resolve(this, absF); 782 } 783 784 /** 785 * Opens an FSDataInputStream at the indicated Path. 786 * 787 * @param f the file name to open 788 * @param bufferSize the size of the buffer to be used. 789 * 790 * @throws AccessControlException If access is denied 791 * @throws FileNotFoundException If file <code>f</code> does not exist 792 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 793 * not supported 794 * @throws IOException If an I/O error occurred 795 * 796 * Exceptions applicable to file systems accessed over RPC: 797 * @throws RpcClientException If an exception occurred in the RPC client 798 * @throws RpcServerException If an exception occurred in the RPC server 799 * @throws UnexpectedServerException If server implementation throws 800 * undeclared exception to RPC server 801 */ 802 public FSDataInputStream open(final Path f, final int bufferSize) 803 throws AccessControlException, FileNotFoundException, 804 UnsupportedFileSystemException, IOException { 805 final Path absF = fixRelativePart(f); 806 return new FSLinkResolver<FSDataInputStream>() { 807 public FSDataInputStream next(final AbstractFileSystem fs, final Path p) 808 throws IOException, UnresolvedLinkException { 809 return fs.open(p, bufferSize); 810 } 811 }.resolve(this, absF); 812 } 813 814 /** 815 * Set replication for an existing file. 816 * 817 * @param f file name 818 * @param replication new replication 819 * 820 * @return true if successful 821 * 822 * @throws AccessControlException If access is denied 823 * @throws FileNotFoundException If file <code>f</code> does not exist 824 * @throws IOException If an I/O error occurred 825 * 826 * Exceptions applicable to file systems accessed over RPC: 827 * @throws RpcClientException If an exception occurred in the RPC client 828 * @throws RpcServerException If an exception occurred in the RPC server 829 * @throws UnexpectedServerException If server implementation throws 830 * undeclared exception to RPC server 831 */ 832 public boolean setReplication(final Path f, final short replication) 833 throws AccessControlException, FileNotFoundException, 834 IOException { 835 final Path absF = fixRelativePart(f); 836 return new FSLinkResolver<Boolean>() { 837 public Boolean next(final AbstractFileSystem fs, final Path p) 838 throws IOException, UnresolvedLinkException { 839 return Boolean.valueOf(fs.setReplication(p, replication)); 840 } 841 }.resolve(this, absF); 842 } 843 844 /** 845 * Renames Path src to Path dst 846 * <ul> 847 * <li 848 * <li>Fails if src is a file and dst is a directory. 849 * <li>Fails if src is a directory and dst is a file. 850 * <li>Fails if the parent of dst does not exist or is a file. 851 * </ul> 852 * <p> 853 * If OVERWRITE option is not passed as an argument, rename fails if the dst 854 * already exists. 855 * <p> 856 * If OVERWRITE option is passed as an argument, rename overwrites the dst if 857 * it is a file or an empty directory. Rename fails if dst is a non-empty 858 * directory. 859 * <p> 860 * Note that atomicity of rename is dependent on the file system 861 * implementation. Please refer to the file system documentation for details 862 * <p> 863 * 864 * @param src path to be renamed 865 * @param dst new path after rename 866 * 867 * @throws AccessControlException If access is denied 868 * @throws FileAlreadyExistsException If <code>dst</code> already exists and 869 * <code>options</options> has {@link Options.Rename#OVERWRITE} 870 * option false. 871 * @throws FileNotFoundException If <code>src</code> does not exist 872 * @throws ParentNotDirectoryException If parent of <code>dst</code> is not a 873 * directory 874 * @throws UnsupportedFileSystemException If file system for <code>src</code> 875 * and <code>dst</code> is not supported 876 * @throws IOException If an I/O error occurred 877 * 878 * Exceptions applicable to file systems accessed over RPC: 879 * @throws RpcClientException If an exception occurred in the RPC client 880 * @throws RpcServerException If an exception occurred in the RPC server 881 * @throws UnexpectedServerException If server implementation throws 882 * undeclared exception to RPC server 883 */ 884 public void rename(final Path src, final Path dst, 885 final Options.Rename... options) throws AccessControlException, 886 FileAlreadyExistsException, FileNotFoundException, 887 ParentNotDirectoryException, UnsupportedFileSystemException, 888 IOException { 889 final Path absSrc = fixRelativePart(src); 890 final Path absDst = fixRelativePart(dst); 891 AbstractFileSystem srcFS = getFSofPath(absSrc); 892 AbstractFileSystem dstFS = getFSofPath(absDst); 893 if(!srcFS.getUri().equals(dstFS.getUri())) { 894 throw new IOException("Renames across AbstractFileSystems not supported"); 895 } 896 try { 897 srcFS.rename(absSrc, absDst, options); 898 } catch (UnresolvedLinkException e) { 899 /* We do not know whether the source or the destination path 900 * was unresolved. Resolve the source path up until the final 901 * path component, then fully resolve the destination. 902 */ 903 final Path source = resolveIntermediate(absSrc); 904 new FSLinkResolver<Void>() { 905 public Void next(final AbstractFileSystem fs, final Path p) 906 throws IOException, UnresolvedLinkException { 907 fs.rename(source, p, options); 908 return null; 909 } 910 }.resolve(this, absDst); 911 } 912 } 913 914 /** 915 * Set permission of a path. 916 * @param f 917 * @param permission - the new absolute permission (umask is not applied) 918 * 919 * @throws AccessControlException If access is denied 920 * @throws FileNotFoundException If <code>f</code> does not exist 921 * @throws UnsupportedFileSystemException If file system for <code>f</code> 922 * is not supported 923 * @throws IOException If an I/O error occurred 924 * 925 * Exceptions applicable to file systems accessed over RPC: 926 * @throws RpcClientException If an exception occurred in the RPC client 927 * @throws RpcServerException If an exception occurred in the RPC server 928 * @throws UnexpectedServerException If server implementation throws 929 * undeclared exception to RPC server 930 */ 931 public void setPermission(final Path f, final FsPermission permission) 932 throws AccessControlException, FileNotFoundException, 933 UnsupportedFileSystemException, IOException { 934 final Path absF = fixRelativePart(f); 935 new FSLinkResolver<Void>() { 936 public Void next(final AbstractFileSystem fs, final Path p) 937 throws IOException, UnresolvedLinkException { 938 fs.setPermission(p, permission); 939 return null; 940 } 941 }.resolve(this, absF); 942 } 943 944 /** 945 * Set owner of a path (i.e. a file or a directory). The parameters username 946 * and groupname cannot both be null. 947 * 948 * @param f The path 949 * @param username If it is null, the original username remains unchanged. 950 * @param groupname If it is null, the original groupname remains unchanged. 951 * 952 * @throws AccessControlException If access is denied 953 * @throws FileNotFoundException If <code>f</code> does not exist 954 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 955 * not supported 956 * @throws IOException If an I/O error occurred 957 * 958 * Exceptions applicable to file systems accessed over RPC: 959 * @throws RpcClientException If an exception occurred in the RPC client 960 * @throws RpcServerException If an exception occurred in the RPC server 961 * @throws UnexpectedServerException If server implementation throws 962 * undeclared exception to RPC server 963 * 964 * RuntimeExceptions: 965 * @throws HadoopIllegalArgumentException If <code>username</code> or 966 * <code>groupname</code> is invalid. 967 */ 968 public void setOwner(final Path f, final String username, 969 final String groupname) throws AccessControlException, 970 UnsupportedFileSystemException, FileNotFoundException, 971 IOException { 972 if ((username == null) && (groupname == null)) { 973 throw new HadoopIllegalArgumentException( 974 "username and groupname cannot both be null"); 975 } 976 final Path absF = fixRelativePart(f); 977 new FSLinkResolver<Void>() { 978 public Void next(final AbstractFileSystem fs, final Path p) 979 throws IOException, UnresolvedLinkException { 980 fs.setOwner(p, username, groupname); 981 return null; 982 } 983 }.resolve(this, absF); 984 } 985 986 /** 987 * Set access time of a file. 988 * @param f The path 989 * @param mtime Set the modification time of this file. 990 * The number of milliseconds since epoch (Jan 1, 1970). 991 * A value of -1 means that this call should not set modification time. 992 * @param atime Set the access time of this file. 993 * The number of milliseconds since Jan 1, 1970. 994 * A value of -1 means that this call should not set access time. 995 * 996 * @throws AccessControlException If access is denied 997 * @throws FileNotFoundException If <code>f</code> does not exist 998 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 999 * not supported 1000 * @throws IOException If an I/O error occurred 1001 * 1002 * Exceptions applicable to file systems accessed over RPC: 1003 * @throws RpcClientException If an exception occurred in the RPC client 1004 * @throws RpcServerException If an exception occurred in the RPC server 1005 * @throws UnexpectedServerException If server implementation throws 1006 * undeclared exception to RPC server 1007 */ 1008 public void setTimes(final Path f, final long mtime, final long atime) 1009 throws AccessControlException, FileNotFoundException, 1010 UnsupportedFileSystemException, IOException { 1011 final Path absF = fixRelativePart(f); 1012 new FSLinkResolver<Void>() { 1013 public Void next(final AbstractFileSystem fs, final Path p) 1014 throws IOException, UnresolvedLinkException { 1015 fs.setTimes(p, mtime, atime); 1016 return null; 1017 } 1018 }.resolve(this, absF); 1019 } 1020 1021 /** 1022 * Get the checksum of a file. 1023 * 1024 * @param f file path 1025 * 1026 * @return The file checksum. The default return value is null, 1027 * which indicates that no checksum algorithm is implemented 1028 * in the corresponding FileSystem. 1029 * 1030 * @throws AccessControlException If access is denied 1031 * @throws FileNotFoundException If <code>f</code> does not exist 1032 * @throws IOException If an I/O error occurred 1033 * 1034 * Exceptions applicable to file systems accessed over RPC: 1035 * @throws RpcClientException If an exception occurred in the RPC client 1036 * @throws RpcServerException If an exception occurred in the RPC server 1037 * @throws UnexpectedServerException If server implementation throws 1038 * undeclared exception to RPC server 1039 */ 1040 public FileChecksum getFileChecksum(final Path f) 1041 throws AccessControlException, FileNotFoundException, 1042 IOException { 1043 final Path absF = fixRelativePart(f); 1044 return new FSLinkResolver<FileChecksum>() { 1045 public FileChecksum next(final AbstractFileSystem fs, final Path p) 1046 throws IOException, UnresolvedLinkException { 1047 return fs.getFileChecksum(p); 1048 } 1049 }.resolve(this, absF); 1050 } 1051 1052 /** 1053 * Set the verify checksum flag for the file system denoted by the path. 1054 * This is only applicable if the 1055 * corresponding FileSystem supports checksum. By default doesn't do anything. 1056 * @param verifyChecksum 1057 * @param f set the verifyChecksum for the Filesystem containing this path 1058 * 1059 * @throws AccessControlException If access is denied 1060 * @throws FileNotFoundException If <code>f</code> does not exist 1061 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1062 * not supported 1063 * @throws IOException If an I/O error occurred 1064 * 1065 * Exceptions applicable to file systems accessed over RPC: 1066 * @throws RpcClientException If an exception occurred in the RPC client 1067 * @throws RpcServerException If an exception occurred in the RPC server 1068 * @throws UnexpectedServerException If server implementation throws 1069 * undeclared exception to RPC server 1070 */ 1071 public void setVerifyChecksum(final boolean verifyChecksum, final Path f) 1072 throws AccessControlException, FileNotFoundException, 1073 UnsupportedFileSystemException, IOException { 1074 final Path absF = resolve(fixRelativePart(f)); 1075 getFSofPath(absF).setVerifyChecksum(verifyChecksum); 1076 } 1077 1078 /** 1079 * Return a file status object that represents the path. 1080 * @param f The path we want information from 1081 * 1082 * @return a FileStatus object 1083 * 1084 * @throws AccessControlException If access is denied 1085 * @throws FileNotFoundException If <code>f</code> does not exist 1086 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1087 * not supported 1088 * @throws IOException If an I/O error occurred 1089 * 1090 * Exceptions applicable to file systems accessed over RPC: 1091 * @throws RpcClientException If an exception occurred in the RPC client 1092 * @throws RpcServerException If an exception occurred in the RPC server 1093 * @throws UnexpectedServerException If server implementation throws 1094 * undeclared exception to RPC server 1095 */ 1096 public FileStatus getFileStatus(final Path f) throws AccessControlException, 1097 FileNotFoundException, UnsupportedFileSystemException, IOException { 1098 final Path absF = fixRelativePart(f); 1099 return new FSLinkResolver<FileStatus>() { 1100 public FileStatus next(final AbstractFileSystem fs, final Path p) 1101 throws IOException, UnresolvedLinkException { 1102 return fs.getFileStatus(p); 1103 } 1104 }.resolve(this, absF); 1105 } 1106 1107 /** 1108 * Return a fully qualified version of the given symlink target if it 1109 * has no scheme and authority. Partially and fully qualified paths 1110 * are returned unmodified. 1111 * @param pathFS The AbstractFileSystem of the path 1112 * @param pathWithLink Path that contains the symlink 1113 * @param target The symlink's absolute target 1114 * @return Fully qualified version of the target. 1115 */ 1116 private Path qualifySymlinkTarget(final AbstractFileSystem pathFS, 1117 Path pathWithLink, Path target) { 1118 // NB: makeQualified uses the target's scheme and authority, if 1119 // specified, and the scheme and authority of pathFS, if not. 1120 final String scheme = target.toUri().getScheme(); 1121 final String auth = target.toUri().getAuthority(); 1122 return (scheme == null && auth == null) 1123 ? target.makeQualified(pathFS.getUri(), pathWithLink.getParent()) 1124 : target; 1125 } 1126 1127 /** 1128 * Return a file status object that represents the path. If the path 1129 * refers to a symlink then the FileStatus of the symlink is returned. 1130 * The behavior is equivalent to #getFileStatus() if the underlying 1131 * file system does not support symbolic links. 1132 * @param f The path we want information from. 1133 * @return A FileStatus object 1134 * 1135 * @throws AccessControlException If access is denied 1136 * @throws FileNotFoundException If <code>f</code> does not exist 1137 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1138 * not supported 1139 * @throws IOException If an I/O error occurred 1140 */ 1141 public FileStatus getFileLinkStatus(final Path f) 1142 throws AccessControlException, FileNotFoundException, 1143 UnsupportedFileSystemException, IOException { 1144 final Path absF = fixRelativePart(f); 1145 return new FSLinkResolver<FileStatus>() { 1146 public FileStatus next(final AbstractFileSystem fs, final Path p) 1147 throws IOException, UnresolvedLinkException { 1148 FileStatus fi = fs.getFileLinkStatus(p); 1149 if (fi.isSymlink()) { 1150 fi.setSymlink(qualifySymlinkTarget(fs, p, fi.getSymlink())); 1151 } 1152 return fi; 1153 } 1154 }.resolve(this, absF); 1155 } 1156 1157 /** 1158 * Returns the target of the given symbolic link as it was specified 1159 * when the link was created. Links in the path leading up to the 1160 * final path component are resolved transparently. 1161 * 1162 * @param f the path to return the target of 1163 * @return The un-interpreted target of the symbolic link. 1164 * 1165 * @throws AccessControlException If access is denied 1166 * @throws FileNotFoundException If path <code>f</code> does not exist 1167 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1168 * not supported 1169 * @throws IOException If the given path does not refer to a symlink 1170 * or an I/O error occurred 1171 */ 1172 public Path getLinkTarget(final Path f) throws AccessControlException, 1173 FileNotFoundException, UnsupportedFileSystemException, IOException { 1174 final Path absF = fixRelativePart(f); 1175 return new FSLinkResolver<Path>() { 1176 public Path next(final AbstractFileSystem fs, final Path p) 1177 throws IOException, UnresolvedLinkException { 1178 FileStatus fi = fs.getFileLinkStatus(p); 1179 return fi.getSymlink(); 1180 } 1181 }.resolve(this, absF); 1182 } 1183 1184 /** 1185 * Return blockLocation of the given file for the given offset and len. 1186 * For a nonexistent file or regions, null will be returned. 1187 * 1188 * This call is most helpful with DFS, where it returns 1189 * hostnames of machines that contain the given file. 1190 * 1191 * @param f - get blocklocations of this file 1192 * @param start position (byte offset) 1193 * @param len (in bytes) 1194 * 1195 * @return block locations for given file at specified offset of len 1196 * 1197 * @throws AccessControlException If access is denied 1198 * @throws FileNotFoundException If <code>f</code> does not exist 1199 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1200 * not supported 1201 * @throws IOException If an I/O error occurred 1202 * 1203 * Exceptions applicable to file systems accessed over RPC: 1204 * @throws RpcClientException If an exception occurred in the RPC client 1205 * @throws RpcServerException If an exception occurred in the RPC server 1206 * @throws UnexpectedServerException If server implementation throws 1207 * undeclared exception to RPC server 1208 * 1209 * RuntimeExceptions: 1210 * @throws InvalidPathException If path <code>f</code> is invalid 1211 */ 1212 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) 1213 @InterfaceStability.Evolving 1214 public BlockLocation[] getFileBlockLocations(final Path f, final long start, 1215 final long len) throws AccessControlException, FileNotFoundException, 1216 UnsupportedFileSystemException, IOException { 1217 final Path absF = fixRelativePart(f); 1218 return new FSLinkResolver<BlockLocation[]>() { 1219 public BlockLocation[] next(final AbstractFileSystem fs, final Path p) 1220 throws IOException, UnresolvedLinkException { 1221 return fs.getFileBlockLocations(p, start, len); 1222 } 1223 }.resolve(this, absF); 1224 } 1225 1226 /** 1227 * Returns a status object describing the use and capacity of the 1228 * file system denoted by the Parh argument p. 1229 * If the file system has multiple partitions, the 1230 * use and capacity of the partition pointed to by the specified 1231 * path is reflected. 1232 * 1233 * @param f Path for which status should be obtained. null means the 1234 * root partition of the default file system. 1235 * 1236 * @return a FsStatus object 1237 * 1238 * @throws AccessControlException If access is denied 1239 * @throws FileNotFoundException If <code>f</code> does not exist 1240 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1241 * not supported 1242 * @throws IOException If an I/O error occurred 1243 * 1244 * Exceptions applicable to file systems accessed over RPC: 1245 * @throws RpcClientException If an exception occurred in the RPC client 1246 * @throws RpcServerException If an exception occurred in the RPC server 1247 * @throws UnexpectedServerException If server implementation throws 1248 * undeclared exception to RPC server 1249 */ 1250 public FsStatus getFsStatus(final Path f) throws AccessControlException, 1251 FileNotFoundException, UnsupportedFileSystemException, IOException { 1252 if (f == null) { 1253 return defaultFS.getFsStatus(); 1254 } 1255 final Path absF = fixRelativePart(f); 1256 return new FSLinkResolver<FsStatus>() { 1257 public FsStatus next(final AbstractFileSystem fs, final Path p) 1258 throws IOException, UnresolvedLinkException { 1259 return fs.getFsStatus(p); 1260 } 1261 }.resolve(this, absF); 1262 } 1263 1264 /** 1265 * Creates a symbolic link to an existing file. An exception is thrown if 1266 * the symlink exits, the user does not have permission to create symlink, 1267 * or the underlying file system does not support symlinks. 1268 * 1269 * Symlink permissions are ignored, access to a symlink is determined by 1270 * the permissions of the symlink target. 1271 * 1272 * Symlinks in paths leading up to the final path component are resolved 1273 * transparently. If the final path component refers to a symlink some 1274 * functions operate on the symlink itself, these are: 1275 * - delete(f) and deleteOnExit(f) - Deletes the symlink. 1276 * - rename(src, dst) - If src refers to a symlink, the symlink is 1277 * renamed. If dst refers to a symlink, the symlink is over-written. 1278 * - getLinkTarget(f) - Returns the target of the symlink. 1279 * - getFileLinkStatus(f) - Returns a FileStatus object describing 1280 * the symlink. 1281 * Some functions, create() and mkdir(), expect the final path component 1282 * does not exist. If they are given a path that refers to a symlink that 1283 * does exist they behave as if the path referred to an existing file or 1284 * directory. All other functions fully resolve, ie follow, the symlink. 1285 * These are: open, setReplication, setOwner, setTimes, setWorkingDirectory, 1286 * setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations, 1287 * getFsStatus, getFileStatus, exists, and listStatus. 1288 * 1289 * Symlink targets are stored as given to createSymlink, assuming the 1290 * underlying file system is capable of storing a fully qualified URI. 1291 * Dangling symlinks are permitted. FileContext supports four types of 1292 * symlink targets, and resolves them as follows 1293 * <pre> 1294 * Given a path referring to a symlink of form: 1295 * 1296 * <---X---> 1297 * fs://host/A/B/link 1298 * <-----Y-----> 1299 * 1300 * In this path X is the scheme and authority that identify the file system, 1301 * and Y is the path leading up to the final path component "link". If Y is 1302 * a symlink itself then let Y' be the target of Y and X' be the scheme and 1303 * authority of Y'. Symlink targets may: 1304 * 1305 * 1. Fully qualified URIs 1306 * 1307 * fs://hostX/A/B/file Resolved according to the target file system. 1308 * 1309 * 2. Partially qualified URIs (eg scheme but no host) 1310 * 1311 * fs:///A/B/file Resolved according to the target file sytem. Eg resolving 1312 * a symlink to hdfs:///A results in an exception because 1313 * HDFS URIs must be fully qualified, while a symlink to 1314 * file:///A will not since Hadoop's local file systems 1315 * require partially qualified URIs. 1316 * 1317 * 3. Relative paths 1318 * 1319 * path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 1320 * is "../B/file" then [Y'][path] is hdfs://host/B/file 1321 * 1322 * 4. Absolute paths 1323 * 1324 * path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path 1325 * is "/file" then [X][path] is hdfs://host/file 1326 * </pre> 1327 * 1328 * @param target the target of the symbolic link 1329 * @param link the path to be created that points to target 1330 * @param createParent if true then missing parent dirs are created if 1331 * false then parent must exist 1332 * 1333 * 1334 * @throws AccessControlException If access is denied 1335 * @throws FileAlreadyExistsException If file <code>linkcode> already exists 1336 * @throws FileNotFoundException If <code>target</code> does not exist 1337 * @throws ParentNotDirectoryException If parent of <code>link</code> is not a 1338 * directory. 1339 * @throws UnsupportedFileSystemException If file system for 1340 * <code>target</code> or <code>link</code> is not supported 1341 * @throws IOException If an I/O error occurred 1342 */ 1343 public void createSymlink(final Path target, final Path link, 1344 final boolean createParent) throws AccessControlException, 1345 FileAlreadyExistsException, FileNotFoundException, 1346 ParentNotDirectoryException, UnsupportedFileSystemException, 1347 IOException { 1348 final Path nonRelLink = fixRelativePart(link); 1349 new FSLinkResolver<Void>() { 1350 public Void next(final AbstractFileSystem fs, final Path p) 1351 throws IOException, UnresolvedLinkException { 1352 fs.createSymlink(target, p, createParent); 1353 return null; 1354 } 1355 }.resolve(this, nonRelLink); 1356 } 1357 1358 /** 1359 * List the statuses of the files/directories in the given path if the path is 1360 * a directory. 1361 * 1362 * @param f is the path 1363 * 1364 * @return an iterator that traverses statuses of the files/directories 1365 * in the given path 1366 * 1367 * @throws AccessControlException If access is denied 1368 * @throws FileNotFoundException If <code>f</code> does not exist 1369 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1370 * not supported 1371 * @throws IOException If an I/O error occurred 1372 * 1373 * Exceptions applicable to file systems accessed over RPC: 1374 * @throws RpcClientException If an exception occurred in the RPC client 1375 * @throws RpcServerException If an exception occurred in the RPC server 1376 * @throws UnexpectedServerException If server implementation throws 1377 * undeclared exception to RPC server 1378 */ 1379 public RemoteIterator<FileStatus> listStatus(final Path f) throws 1380 AccessControlException, FileNotFoundException, 1381 UnsupportedFileSystemException, IOException { 1382 final Path absF = fixRelativePart(f); 1383 return new FSLinkResolver<RemoteIterator<FileStatus>>() { 1384 public RemoteIterator<FileStatus> next( 1385 final AbstractFileSystem fs, final Path p) 1386 throws IOException, UnresolvedLinkException { 1387 return fs.listStatusIterator(p); 1388 } 1389 }.resolve(this, absF); 1390 } 1391 1392 /** 1393 * @return an iterator over the corrupt files under the given path 1394 * (may contain duplicates if a file has more than one corrupt block) 1395 * @throws IOException 1396 */ 1397 public RemoteIterator<Path> listCorruptFileBlocks(Path path) 1398 throws IOException { 1399 final Path absF = fixRelativePart(path); 1400 return new FSLinkResolver<RemoteIterator<Path>>() { 1401 @Override 1402 public RemoteIterator<Path> next(final AbstractFileSystem fs, 1403 final Path p) 1404 throws IOException, UnresolvedLinkException { 1405 return fs.listCorruptFileBlocks(p); 1406 } 1407 }.resolve(this, absF); 1408 } 1409 1410 /** 1411 * List the statuses of the files/directories in the given path if the path is 1412 * a directory. 1413 * Return the file's status and block locations If the path is a file. 1414 * 1415 * If a returned status is a file, it contains the file's block locations. 1416 * 1417 * @param f is the path 1418 * 1419 * @return an iterator that traverses statuses of the files/directories 1420 * in the given path 1421 * If any IO exception (for example the input directory gets deleted while 1422 * listing is being executed), next() or hasNext() of the returned iterator 1423 * may throw a RuntimeException with the io exception as the cause. 1424 * 1425 * @throws AccessControlException If access is denied 1426 * @throws FileNotFoundException If <code>f</code> does not exist 1427 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1428 * not supported 1429 * @throws IOException If an I/O error occurred 1430 * 1431 * Exceptions applicable to file systems accessed over RPC: 1432 * @throws RpcClientException If an exception occurred in the RPC client 1433 * @throws RpcServerException If an exception occurred in the RPC server 1434 * @throws UnexpectedServerException If server implementation throws 1435 * undeclared exception to RPC server 1436 */ 1437 public RemoteIterator<LocatedFileStatus> listLocatedStatus( 1438 final Path f) throws 1439 AccessControlException, FileNotFoundException, 1440 UnsupportedFileSystemException, IOException { 1441 final Path absF = fixRelativePart(f); 1442 return new FSLinkResolver<RemoteIterator<LocatedFileStatus>>() { 1443 public RemoteIterator<LocatedFileStatus> next( 1444 final AbstractFileSystem fs, final Path p) 1445 throws IOException, UnresolvedLinkException { 1446 return fs.listLocatedStatus(p); 1447 } 1448 }.resolve(this, absF); 1449 } 1450 1451 /** 1452 * Mark a path to be deleted on JVM shutdown. 1453 * 1454 * @param f the existing path to delete. 1455 * 1456 * @return true if deleteOnExit is successful, otherwise false. 1457 * 1458 * @throws AccessControlException If access is denied 1459 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1460 * not supported 1461 * @throws IOException If an I/O error occurred 1462 * 1463 * Exceptions applicable to file systems accessed over RPC: 1464 * @throws RpcClientException If an exception occurred in the RPC client 1465 * @throws RpcServerException If an exception occurred in the RPC server 1466 * @throws UnexpectedServerException If server implementation throws 1467 * undeclared exception to RPC server 1468 */ 1469 public boolean deleteOnExit(Path f) throws AccessControlException, 1470 IOException { 1471 if (!this.util().exists(f)) { 1472 return false; 1473 } 1474 synchronized (DELETE_ON_EXIT) { 1475 if (DELETE_ON_EXIT.isEmpty()) { 1476 ShutdownHookManager.get().addShutdownHook(FINALIZER, SHUTDOWN_HOOK_PRIORITY); 1477 } 1478 1479 Set<Path> set = DELETE_ON_EXIT.get(this); 1480 if (set == null) { 1481 set = new TreeSet<Path>(); 1482 DELETE_ON_EXIT.put(this, set); 1483 } 1484 set.add(f); 1485 } 1486 return true; 1487 } 1488 1489 private final Util util; 1490 public Util util() { 1491 return util; 1492 } 1493 1494 1495 /** 1496 * Utility/library methods built over the basic FileContext methods. 1497 * Since this are library functions, the oprtation are not atomic 1498 * and some of them may partially complete if other threads are making 1499 * changes to the same part of the name space. 1500 */ 1501 public class Util { 1502 /** 1503 * Does the file exist? 1504 * Note: Avoid using this method if you already have FileStatus in hand. 1505 * Instead reuse the FileStatus 1506 * @param f the file or dir to be checked 1507 * 1508 * @throws AccessControlException If access is denied 1509 * @throws IOException If an I/O error occurred 1510 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1511 * not supported 1512 * 1513 * Exceptions applicable to file systems accessed over RPC: 1514 * @throws RpcClientException If an exception occurred in the RPC client 1515 * @throws RpcServerException If an exception occurred in the RPC server 1516 * @throws UnexpectedServerException If server implementation throws 1517 * undeclared exception to RPC server 1518 */ 1519 public boolean exists(final Path f) throws AccessControlException, 1520 UnsupportedFileSystemException, IOException { 1521 try { 1522 FileStatus fs = FileContext.this.getFileStatus(f); 1523 assert fs != null; 1524 return true; 1525 } catch (FileNotFoundException e) { 1526 return false; 1527 } 1528 } 1529 1530 /** 1531 * Return a list of file status objects that corresponds to supplied paths 1532 * excluding those non-existent paths. 1533 * 1534 * @param paths list of paths we want information from 1535 * 1536 * @return a list of FileStatus objects 1537 * 1538 * @throws AccessControlException If access is denied 1539 * @throws IOException If an I/O error occurred 1540 * 1541 * Exceptions applicable to file systems accessed over RPC: 1542 * @throws RpcClientException If an exception occurred in the RPC client 1543 * @throws RpcServerException If an exception occurred in the RPC server 1544 * @throws UnexpectedServerException If server implementation throws 1545 * undeclared exception to RPC server 1546 */ 1547 private FileStatus[] getFileStatus(Path[] paths) 1548 throws AccessControlException, IOException { 1549 if (paths == null) { 1550 return null; 1551 } 1552 ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length); 1553 for (int i = 0; i < paths.length; i++) { 1554 try { 1555 results.add(FileContext.this.getFileStatus(paths[i])); 1556 } catch (FileNotFoundException fnfe) { 1557 // ignoring 1558 } 1559 } 1560 return results.toArray(new FileStatus[results.size()]); 1561 } 1562 1563 1564 /** 1565 * Return the {@link ContentSummary} of path f. 1566 * @param f path 1567 * 1568 * @return the {@link ContentSummary} of path f. 1569 * 1570 * @throws AccessControlException If access is denied 1571 * @throws FileNotFoundException If <code>f</code> does not exist 1572 * @throws UnsupportedFileSystemException If file system for 1573 * <code>f</code> is not supported 1574 * @throws IOException If an I/O error occurred 1575 * 1576 * Exceptions applicable to file systems accessed over RPC: 1577 * @throws RpcClientException If an exception occurred in the RPC client 1578 * @throws RpcServerException If an exception occurred in the RPC server 1579 * @throws UnexpectedServerException If server implementation throws 1580 * undeclared exception to RPC server 1581 */ 1582 public ContentSummary getContentSummary(Path f) 1583 throws AccessControlException, FileNotFoundException, 1584 UnsupportedFileSystemException, IOException { 1585 FileStatus status = FileContext.this.getFileStatus(f); 1586 if (status.isFile()) { 1587 return new ContentSummary(status.getLen(), 1, 0); 1588 } 1589 long[] summary = {0, 0, 1}; 1590 RemoteIterator<FileStatus> statusIterator = 1591 FileContext.this.listStatus(f); 1592 while(statusIterator.hasNext()) { 1593 FileStatus s = statusIterator.next(); 1594 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : 1595 new ContentSummary(s.getLen(), 1, 0); 1596 summary[0] += c.getLength(); 1597 summary[1] += c.getFileCount(); 1598 summary[2] += c.getDirectoryCount(); 1599 } 1600 return new ContentSummary(summary[0], summary[1], summary[2]); 1601 } 1602 1603 /** 1604 * See {@link #listStatus(Path[], PathFilter)} 1605 */ 1606 public FileStatus[] listStatus(Path[] files) throws AccessControlException, 1607 FileNotFoundException, IOException { 1608 return listStatus(files, DEFAULT_FILTER); 1609 } 1610 1611 /** 1612 * Filter files/directories in the given path using the user-supplied path 1613 * filter. 1614 * 1615 * @param f is the path name 1616 * @param filter is the user-supplied path filter 1617 * 1618 * @return an array of FileStatus objects for the files under the given path 1619 * after applying the filter 1620 * 1621 * @throws AccessControlException If access is denied 1622 * @throws FileNotFoundException If <code>f</code> does not exist 1623 * @throws UnsupportedFileSystemException If file system for 1624 * <code>pathPattern</code> is not supported 1625 * @throws IOException If an I/O error occurred 1626 * 1627 * Exceptions applicable to file systems accessed over RPC: 1628 * @throws RpcClientException If an exception occurred in the RPC client 1629 * @throws RpcServerException If an exception occurred in the RPC server 1630 * @throws UnexpectedServerException If server implementation throws 1631 * undeclared exception to RPC server 1632 */ 1633 public FileStatus[] listStatus(Path f, PathFilter filter) 1634 throws AccessControlException, FileNotFoundException, 1635 UnsupportedFileSystemException, IOException { 1636 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1637 listStatus(results, f, filter); 1638 return results.toArray(new FileStatus[results.size()]); 1639 } 1640 1641 /** 1642 * Filter files/directories in the given list of paths using user-supplied 1643 * path filter. 1644 * 1645 * @param files is a list of paths 1646 * @param filter is the filter 1647 * 1648 * @return a list of statuses for the files under the given paths after 1649 * applying the filter 1650 * 1651 * @throws AccessControlException If access is denied 1652 * @throws FileNotFoundException If a file in <code>files</code> does not 1653 * exist 1654 * @throws IOException If an I/O error occurred 1655 * 1656 * Exceptions applicable to file systems accessed over RPC: 1657 * @throws RpcClientException If an exception occurred in the RPC client 1658 * @throws RpcServerException If an exception occurred in the RPC server 1659 * @throws UnexpectedServerException If server implementation throws 1660 * undeclared exception to RPC server 1661 */ 1662 public FileStatus[] listStatus(Path[] files, PathFilter filter) 1663 throws AccessControlException, FileNotFoundException, IOException { 1664 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1665 for (int i = 0; i < files.length; i++) { 1666 listStatus(results, files[i], filter); 1667 } 1668 return results.toArray(new FileStatus[results.size()]); 1669 } 1670 1671 /* 1672 * Filter files/directories in the given path using the user-supplied path 1673 * filter. Results are added to the given array <code>results</code>. 1674 */ 1675 private void listStatus(ArrayList<FileStatus> results, Path f, 1676 PathFilter filter) throws AccessControlException, 1677 FileNotFoundException, IOException { 1678 FileStatus[] listing = listStatus(f); 1679 if (listing != null) { 1680 for (int i = 0; i < listing.length; i++) { 1681 if (filter.accept(listing[i].getPath())) { 1682 results.add(listing[i]); 1683 } 1684 } 1685 } 1686 } 1687 1688 /** 1689 * List the statuses of the files/directories in the given path 1690 * if the path is a directory. 1691 * 1692 * @param f is the path 1693 * 1694 * @return an array that contains statuses of the files/directories 1695 * in the given path 1696 * 1697 * @throws AccessControlException If access is denied 1698 * @throws FileNotFoundException If <code>f</code> does not exist 1699 * @throws UnsupportedFileSystemException If file system for <code>f</code> is 1700 * not supported 1701 * @throws IOException If an I/O error occurred 1702 * 1703 * Exceptions applicable to file systems accessed over RPC: 1704 * @throws RpcClientException If an exception occurred in the RPC client 1705 * @throws RpcServerException If an exception occurred in the RPC server 1706 * @throws UnexpectedServerException If server implementation throws 1707 * undeclared exception to RPC server 1708 */ 1709 public FileStatus[] listStatus(final Path f) throws AccessControlException, 1710 FileNotFoundException, UnsupportedFileSystemException, 1711 IOException { 1712 final Path absF = fixRelativePart(f); 1713 return new FSLinkResolver<FileStatus[]>() { 1714 public FileStatus[] next(final AbstractFileSystem fs, final Path p) 1715 throws IOException, UnresolvedLinkException { 1716 return fs.listStatus(p); 1717 } 1718 }.resolve(FileContext.this, absF); 1719 } 1720 1721 /** 1722 * List the statuses and block locations of the files in the given path. 1723 * 1724 * If the path is a directory, 1725 * if recursive is false, returns files in the directory; 1726 * if recursive is true, return files in the subtree rooted at the path. 1727 * The subtree is traversed in the depth-first order. 1728 * If the path is a file, return the file's status and block locations. 1729 * Files across symbolic links are also returned. 1730 * 1731 * @param f is the path 1732 * @param recursive if the subdirectories need to be traversed recursively 1733 * 1734 * @return an iterator that traverses statuses of the files 1735 * If any IO exception (for example a sub-directory gets deleted while 1736 * listing is being executed), next() or hasNext() of the returned iterator 1737 * may throw a RuntimeException with the IO exception as the cause. 1738 * 1739 * @throws AccessControlException If access is denied 1740 * @throws FileNotFoundException If <code>f</code> does not exist 1741 * @throws UnsupportedFileSystemException If file system for <code>f</code> 1742 * is not supported 1743 * @throws IOException If an I/O error occurred 1744 * 1745 * Exceptions applicable to file systems accessed over RPC: 1746 * @throws RpcClientException If an exception occurred in the RPC client 1747 * @throws RpcServerException If an exception occurred in the RPC server 1748 * @throws UnexpectedServerException If server implementation throws 1749 * undeclared exception to RPC server 1750 */ 1751 public RemoteIterator<LocatedFileStatus> listFiles( 1752 final Path f, final boolean recursive) throws AccessControlException, 1753 FileNotFoundException, UnsupportedFileSystemException, 1754 IOException { 1755 return new RemoteIterator<LocatedFileStatus>() { 1756 private Stack<RemoteIterator<LocatedFileStatus>> itors = 1757 new Stack<RemoteIterator<LocatedFileStatus>>(); 1758 RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f); 1759 LocatedFileStatus curFile; 1760 1761 /** 1762 * Returns <tt>true</tt> if the iterator has more files. 1763 * 1764 * @return <tt>true</tt> if the iterator has more files. 1765 * @throws AccessControlException if not allowed to access next 1766 * file's status or locations 1767 * @throws FileNotFoundException if next file does not exist any more 1768 * @throws UnsupportedFileSystemException if next file's 1769 * fs is unsupported 1770 * @throws IOException for all other IO errors 1771 * for example, NameNode is not avaialbe or 1772 * NameNode throws IOException due to an error 1773 * while getting the status or block locations 1774 */ 1775 @Override 1776 public boolean hasNext() throws IOException { 1777 while (curFile == null) { 1778 if (curItor.hasNext()) { 1779 handleFileStat(curItor.next()); 1780 } else if (!itors.empty()) { 1781 curItor = itors.pop(); 1782 } else { 1783 return false; 1784 } 1785 } 1786 return true; 1787 } 1788 1789 /** 1790 * Process the input stat. 1791 * If it is a file, return the file stat. 1792 * If it is a directory, traverse the directory if recursive is true; 1793 * ignore it if recursive is false. 1794 * If it is a symlink, resolve the symlink first and then process it 1795 * depending on if it is a file or directory. 1796 * @param stat input status 1797 * @throws AccessControlException if access is denied 1798 * @throws FileNotFoundException if file is not found 1799 * @throws UnsupportedFileSystemException if fs is not supported 1800 * @throws IOException for all other IO errors 1801 */ 1802 private void handleFileStat(LocatedFileStatus stat) 1803 throws IOException { 1804 if (stat.isFile()) { // file 1805 curFile = stat; 1806 } else if (stat.isSymlink()) { // symbolic link 1807 // resolve symbolic link 1808 FileStatus symstat = FileContext.this.getFileStatus( 1809 stat.getSymlink()); 1810 if (symstat.isFile() || (recursive && symstat.isDirectory())) { 1811 itors.push(curItor); 1812 curItor = listLocatedStatus(stat.getPath()); 1813 } 1814 } else if (recursive) { // directory 1815 itors.push(curItor); 1816 curItor = listLocatedStatus(stat.getPath()); 1817 } 1818 } 1819 1820 /** 1821 * Returns the next file's status with its block locations 1822 * 1823 * @throws AccessControlException if not allowed to access next 1824 * file's status or locations 1825 * @throws FileNotFoundException if next file does not exist any more 1826 * @throws UnsupportedFileSystemException if next file's 1827 * fs is unsupported 1828 * @throws IOException for all other IO errors 1829 * for example, NameNode is not avaialbe or 1830 * NameNode throws IOException due to an error 1831 * while getting the status or block locations 1832 */ 1833 @Override 1834 public LocatedFileStatus next() throws IOException { 1835 if (hasNext()) { 1836 LocatedFileStatus result = curFile; 1837 curFile = null; 1838 return result; 1839 } 1840 throw new java.util.NoSuchElementException("No more entry in " + f); 1841 } 1842 }; 1843 } 1844 1845 /** 1846 * <p>Return all the files that match filePattern and are not checksum 1847 * files. Results are sorted by their names. 1848 * 1849 * <p> 1850 * A filename pattern is composed of <i>regular</i> characters and 1851 * <i>special pattern matching</i> characters, which are: 1852 * 1853 * <dl> 1854 * <dd> 1855 * <dl> 1856 * <p> 1857 * <dt> <tt> ? </tt> 1858 * <dd> Matches any single character. 1859 * 1860 * <p> 1861 * <dt> <tt> * </tt> 1862 * <dd> Matches zero or more characters. 1863 * 1864 * <p> 1865 * <dt> <tt> [<i>abc</i>] </tt> 1866 * <dd> Matches a single character from character set 1867 * <tt>{<i>a,b,c</i>}</tt>. 1868 * 1869 * <p> 1870 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> 1871 * <dd> Matches a single character from the character range 1872 * <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be 1873 * lexicographically less than or equal to character <tt><i>b</i></tt>. 1874 * 1875 * <p> 1876 * <dt> <tt> [^<i>a</i>] </tt> 1877 * <dd> Matches a single char that is not from character set or range 1878 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur 1879 * immediately to the right of the opening bracket. 1880 * 1881 * <p> 1882 * <dt> <tt> \<i>c</i> </tt> 1883 * <dd> Removes (escapes) any special meaning of character <i>c</i>. 1884 * 1885 * <p> 1886 * <dt> <tt> {ab,cd} </tt> 1887 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> 1888 * 1889 * <p> 1890 * <dt> <tt> {ab,c{de,fh}} </tt> 1891 * <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt> 1892 * 1893 * </dl> 1894 * </dd> 1895 * </dl> 1896 * 1897 * @param pathPattern a regular expression specifying a pth pattern 1898 * 1899 * @return an array of paths that match the path pattern 1900 * 1901 * @throws AccessControlException If access is denied 1902 * @throws UnsupportedFileSystemException If file system for 1903 * <code>pathPattern</code> is not supported 1904 * @throws IOException If an I/O error occurred 1905 * 1906 * Exceptions applicable to file systems accessed over RPC: 1907 * @throws RpcClientException If an exception occurred in the RPC client 1908 * @throws RpcServerException If an exception occurred in the RPC server 1909 * @throws UnexpectedServerException If server implementation throws 1910 * undeclared exception to RPC server 1911 */ 1912 public FileStatus[] globStatus(Path pathPattern) 1913 throws AccessControlException, UnsupportedFileSystemException, 1914 IOException { 1915 return globStatus(pathPattern, DEFAULT_FILTER); 1916 } 1917 1918 /** 1919 * Return an array of FileStatus objects whose path names match pathPattern 1920 * and is accepted by the user-supplied path filter. Results are sorted by 1921 * their path names. 1922 * Return null if pathPattern has no glob and the path does not exist. 1923 * Return an empty array if pathPattern has a glob and no path matches it. 1924 * 1925 * @param pathPattern regular expression specifying the path pattern 1926 * @param filter user-supplied path filter 1927 * 1928 * @return an array of FileStatus objects 1929 * 1930 * @throws AccessControlException If access is denied 1931 * @throws UnsupportedFileSystemException If file system for 1932 * <code>pathPattern</code> is not supported 1933 * @throws IOException If an I/O error occurred 1934 * 1935 * Exceptions applicable to file systems accessed over RPC: 1936 * @throws RpcClientException If an exception occurred in the RPC client 1937 * @throws RpcServerException If an exception occurred in the RPC server 1938 * @throws UnexpectedServerException If server implementation throws 1939 * undeclared exception to RPC server 1940 */ 1941 public FileStatus[] globStatus(final Path pathPattern, 1942 final PathFilter filter) throws AccessControlException, 1943 UnsupportedFileSystemException, IOException { 1944 URI uri = getFSofPath(fixRelativePart(pathPattern)).getUri(); 1945 1946 String filename = pathPattern.toUri().getPath(); 1947 1948 List<String> filePatterns = GlobExpander.expand(filename); 1949 if (filePatterns.size() == 1) { 1950 Path absPathPattern = fixRelativePart(pathPattern); 1951 return globStatusInternal(uri, new Path(absPathPattern.toUri() 1952 .getPath()), filter); 1953 } else { 1954 List<FileStatus> results = new ArrayList<FileStatus>(); 1955 for (String iFilePattern : filePatterns) { 1956 Path iAbsFilePattern = fixRelativePart(new Path(iFilePattern)); 1957 FileStatus[] files = globStatusInternal(uri, iAbsFilePattern, filter); 1958 for (FileStatus file : files) { 1959 results.add(file); 1960 } 1961 } 1962 return results.toArray(new FileStatus[results.size()]); 1963 } 1964 } 1965 1966 /** 1967 * 1968 * @param uri for all the inPathPattern 1969 * @param inPathPattern - without the scheme & authority (take from uri) 1970 * @param filter 1971 * 1972 * @return an array of FileStatus objects 1973 * 1974 * @throws AccessControlException If access is denied 1975 * @throws IOException If an I/O error occurred 1976 */ 1977 private FileStatus[] globStatusInternal(final URI uri, 1978 final Path inPathPattern, final PathFilter filter) 1979 throws AccessControlException, IOException 1980 { 1981 Path[] parents = new Path[1]; 1982 int level = 0; 1983 1984 assert(inPathPattern.toUri().getScheme() == null && 1985 inPathPattern.toUri().getAuthority() == null && 1986 inPathPattern.isUriPathAbsolute()); 1987 1988 1989 String filename = inPathPattern.toUri().getPath(); 1990 1991 // path has only zero component 1992 if ("".equals(filename) || Path.SEPARATOR.equals(filename)) { 1993 Path p = inPathPattern.makeQualified(uri, null); 1994 return getFileStatus(new Path[]{p}); 1995 } 1996 1997 // path has at least one component 1998 String[] components = filename.split(Path.SEPARATOR); 1999 2000 // Path is absolute, first component is "/" hence first component 2001 // is the uri root 2002 parents[0] = new Path(new Path(uri), new Path("/")); 2003 level = 1; 2004 2005 // glob the paths that match the parent path, ie. [0, components.length-1] 2006 boolean[] hasGlob = new boolean[]{false}; 2007 Path[] relParentPaths = 2008 globPathsLevel(parents, components, level, hasGlob); 2009 FileStatus[] results; 2010 2011 if (relParentPaths == null || relParentPaths.length == 0) { 2012 results = null; 2013 } else { 2014 // fix the pathes to be abs 2015 Path[] parentPaths = new Path [relParentPaths.length]; 2016 for(int i=0; i<relParentPaths.length; i++) { 2017 parentPaths[i] = relParentPaths[i].makeQualified(uri, null); 2018 } 2019 2020 // Now work on the last component of the path 2021 GlobFilter fp = 2022 new GlobFilter(components[components.length - 1], filter); 2023 if (fp.hasPattern()) { // last component has a pattern 2024 // list parent directories and then glob the results 2025 try { 2026 results = listStatus(parentPaths, fp); 2027 } catch (FileNotFoundException e) { 2028 results = null; 2029 } 2030 hasGlob[0] = true; 2031 } else { // last component does not have a pattern 2032 // get all the path names 2033 ArrayList<Path> filteredPaths = 2034 new ArrayList<Path>(parentPaths.length); 2035 for (int i = 0; i < parentPaths.length; i++) { 2036 parentPaths[i] = new Path(parentPaths[i], 2037 components[components.length - 1]); 2038 if (fp.accept(parentPaths[i])) { 2039 filteredPaths.add(parentPaths[i]); 2040 } 2041 } 2042 // get all their statuses 2043 results = getFileStatus( 2044 filteredPaths.toArray(new Path[filteredPaths.size()])); 2045 } 2046 } 2047 2048 // Decide if the pathPattern contains a glob or not 2049 if (results == null) { 2050 if (hasGlob[0]) { 2051 results = new FileStatus[0]; 2052 } 2053 } else { 2054 if (results.length == 0) { 2055 if (!hasGlob[0]) { 2056 results = null; 2057 } 2058 } else { 2059 Arrays.sort(results); 2060 } 2061 } 2062 return results; 2063 } 2064 2065 /* 2066 * For a path of N components, return a list of paths that match the 2067 * components [<code>level</code>, <code>N-1</code>]. 2068 */ 2069 private Path[] globPathsLevel(Path[] parents, String[] filePattern, 2070 int level, boolean[] hasGlob) throws AccessControlException, 2071 FileNotFoundException, IOException { 2072 if (level == filePattern.length - 1) { 2073 return parents; 2074 } 2075 if (parents == null || parents.length == 0) { 2076 return null; 2077 } 2078 GlobFilter fp = new GlobFilter(filePattern[level]); 2079 if (fp.hasPattern()) { 2080 try { 2081 parents = FileUtil.stat2Paths(listStatus(parents, fp)); 2082 } catch (FileNotFoundException e) { 2083 parents = null; 2084 } 2085 hasGlob[0] = true; 2086 } else { 2087 for (int i = 0; i < parents.length; i++) { 2088 parents[i] = new Path(parents[i], filePattern[level]); 2089 } 2090 } 2091 return globPathsLevel(parents, filePattern, level + 1, hasGlob); 2092 } 2093 2094 /** 2095 * Copy file from src to dest. See 2096 * {@link #copy(Path, Path, boolean, boolean)} 2097 */ 2098 public boolean copy(final Path src, final Path dst) 2099 throws AccessControlException, FileAlreadyExistsException, 2100 FileNotFoundException, ParentNotDirectoryException, 2101 UnsupportedFileSystemException, IOException { 2102 return copy(src, dst, false, false); 2103 } 2104 2105 /** 2106 * Copy from src to dst, optionally deleting src and overwriting dst. 2107 * @param src 2108 * @param dst 2109 * @param deleteSource - delete src if true 2110 * @param overwrite overwrite dst if true; throw IOException if dst exists 2111 * and overwrite is false. 2112 * 2113 * @return true if copy is successful 2114 * 2115 * @throws AccessControlException If access is denied 2116 * @throws FileAlreadyExistsException If <code>dst</code> already exists 2117 * @throws FileNotFoundException If <code>src</code> does not exist 2118 * @throws ParentNotDirectoryException If parent of <code>dst</code> is not 2119 * a directory 2120 * @throws UnsupportedFileSystemException If file system for 2121 * <code>src</code> or <code>dst</code> is not supported 2122 * @throws IOException If an I/O error occurred 2123 * 2124 * Exceptions applicable to file systems accessed over RPC: 2125 * @throws RpcClientException If an exception occurred in the RPC client 2126 * @throws RpcServerException If an exception occurred in the RPC server 2127 * @throws UnexpectedServerException If server implementation throws 2128 * undeclared exception to RPC server 2129 * 2130 * RuntimeExceptions: 2131 * @throws InvalidPathException If path <code>dst</code> is invalid 2132 */ 2133 public boolean copy(final Path src, final Path dst, boolean deleteSource, 2134 boolean overwrite) throws AccessControlException, 2135 FileAlreadyExistsException, FileNotFoundException, 2136 ParentNotDirectoryException, UnsupportedFileSystemException, 2137 IOException { 2138 checkNotSchemeWithRelative(src); 2139 checkNotSchemeWithRelative(dst); 2140 Path qSrc = makeQualified(src); 2141 Path qDst = makeQualified(dst); 2142 checkDest(qSrc.getName(), qDst, overwrite); 2143 FileStatus fs = FileContext.this.getFileStatus(qSrc); 2144 if (fs.isDirectory()) { 2145 checkDependencies(qSrc, qDst); 2146 mkdir(qDst, FsPermission.getDirDefault(), true); 2147 FileStatus[] contents = listStatus(qSrc); 2148 for (FileStatus content : contents) { 2149 copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, 2150 content.getPath().getName())), deleteSource, overwrite); 2151 } 2152 } else { 2153 InputStream in=null; 2154 OutputStream out = null; 2155 try { 2156 in = open(qSrc); 2157 EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of( 2158 CreateFlag.CREATE, CreateFlag.OVERWRITE) : 2159 EnumSet.of(CreateFlag.CREATE); 2160 out = create(qDst, createFlag); 2161 IOUtils.copyBytes(in, out, conf, true); 2162 } catch (IOException e) { 2163 IOUtils.closeStream(out); 2164 IOUtils.closeStream(in); 2165 throw e; 2166 } 2167 } 2168 if (deleteSource) { 2169 return delete(qSrc, true); 2170 } else { 2171 return true; 2172 } 2173 } 2174 } 2175 2176 /** 2177 * Check if copying srcName to dst would overwrite an existing 2178 * file or directory. 2179 * @param srcName File or directory to be copied. 2180 * @param dst Destination to copy srcName to. 2181 * @param overwrite Whether it's ok to overwrite an existing file. 2182 * @throws AccessControlException If access is denied. 2183 * @throws IOException If dst is an existing directory, or dst is an 2184 * existing file and the overwrite option is not passed. 2185 */ 2186 private void checkDest(String srcName, Path dst, boolean overwrite) 2187 throws AccessControlException, IOException { 2188 try { 2189 FileStatus dstFs = getFileStatus(dst); 2190 if (dstFs.isDirectory()) { 2191 if (null == srcName) { 2192 throw new IOException("Target " + dst + " is a directory"); 2193 } 2194 // Recurse to check if dst/srcName exists. 2195 checkDest(null, new Path(dst, srcName), overwrite); 2196 } else if (!overwrite) { 2197 throw new IOException("Target " + new Path(dst, srcName) 2198 + " already exists"); 2199 } 2200 } catch (FileNotFoundException e) { 2201 // dst does not exist - OK to copy. 2202 } 2203 } 2204 2205 // 2206 // If the destination is a subdirectory of the source, then 2207 // generate exception 2208 // 2209 private static void checkDependencies(Path qualSrc, Path qualDst) 2210 throws IOException { 2211 if (isSameFS(qualSrc, qualDst)) { 2212 String srcq = qualSrc.toString() + Path.SEPARATOR; 2213 String dstq = qualDst.toString() + Path.SEPARATOR; 2214 if (dstq.startsWith(srcq)) { 2215 if (srcq.length() == dstq.length()) { 2216 throw new IOException("Cannot copy " + qualSrc + " to itself."); 2217 } else { 2218 throw new IOException("Cannot copy " + qualSrc + 2219 " to its subdirectory " + qualDst); 2220 } 2221 } 2222 } 2223 } 2224 2225 /** 2226 * Are qualSrc and qualDst of the same file system? 2227 * @param qualPath1 - fully qualified path 2228 * @param qualPath2 - fully qualified path 2229 * @return 2230 */ 2231 private static boolean isSameFS(Path qualPath1, Path qualPath2) { 2232 URI srcUri = qualPath1.toUri(); 2233 URI dstUri = qualPath2.toUri(); 2234 return (srcUri.getScheme().equals(dstUri.getScheme()) && 2235 !(srcUri.getAuthority() != null && dstUri.getAuthority() != null && srcUri 2236 .getAuthority().equals(dstUri.getAuthority()))); 2237 } 2238 2239 /** 2240 * Deletes all the paths in deleteOnExit on JVM shutdown. 2241 */ 2242 static class FileContextFinalizer implements Runnable { 2243 public synchronized void run() { 2244 processDeleteOnExit(); 2245 } 2246 } 2247 2248 /** 2249 * Resolves all symbolic links in the specified path. 2250 * Returns the new path object. 2251 */ 2252 protected Path resolve(final Path f) throws FileNotFoundException, 2253 UnresolvedLinkException, AccessControlException, IOException { 2254 return new FSLinkResolver<Path>() { 2255 public Path next(final AbstractFileSystem fs, final Path p) 2256 throws IOException, UnresolvedLinkException { 2257 return fs.resolvePath(p); 2258 } 2259 }.resolve(this, f); 2260 } 2261 2262 /** 2263 * Resolves all symbolic links in the specified path leading up 2264 * to, but not including the final path component. 2265 * @param f path to resolve 2266 * @return the new path object. 2267 */ 2268 protected Path resolveIntermediate(final Path f) throws IOException { 2269 return new FSLinkResolver<FileStatus>() { 2270 public FileStatus next(final AbstractFileSystem fs, final Path p) 2271 throws IOException, UnresolvedLinkException { 2272 return fs.getFileLinkStatus(p); 2273 } 2274 }.resolve(this, f).getPath(); 2275 } 2276 2277 /** 2278 * Returns the list of AbstractFileSystems accessed in the path. The list may 2279 * contain more than one AbstractFileSystems objects in case of symlinks. 2280 * 2281 * @param f 2282 * Path which needs to be resolved 2283 * @return List of AbstractFileSystems accessed in the path 2284 * @throws IOException 2285 */ 2286 Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f) 2287 throws IOException { 2288 final Path absF = fixRelativePart(f); 2289 final HashSet<AbstractFileSystem> result 2290 = new HashSet<AbstractFileSystem>(); 2291 new FSLinkResolver<Void>() { 2292 public Void next(final AbstractFileSystem fs, final Path p) 2293 throws IOException, UnresolvedLinkException { 2294 result.add(fs); 2295 fs.getFileStatus(p); 2296 return null; 2297 } 2298 }.resolve(this, absF); 2299 return result; 2300 } 2301 2302 /** 2303 * Class used to perform an operation on and resolve symlinks in a 2304 * path. The operation may potentially span multiple file systems. 2305 */ 2306 protected abstract class FSLinkResolver<T> { 2307 // The maximum number of symbolic link components in a path 2308 private static final int MAX_PATH_LINKS = 32; 2309 2310 /** 2311 * Generic helper function overridden on instantiation to perform a 2312 * specific operation on the given file system using the given path 2313 * which may result in an UnresolvedLinkException. 2314 * @param fs AbstractFileSystem to perform the operation on. 2315 * @param p Path given the file system. 2316 * @return Generic type determined by the specific implementation. 2317 * @throws UnresolvedLinkException If symbolic link <code>path</code> could 2318 * not be resolved 2319 * @throws IOException an I/O error occured 2320 */ 2321 public abstract T next(final AbstractFileSystem fs, final Path p) 2322 throws IOException, UnresolvedLinkException; 2323 2324 /** 2325 * Performs the operation specified by the next function, calling it 2326 * repeatedly until all symlinks in the given path are resolved. 2327 * @param fc FileContext used to access file systems. 2328 * @param p The path to resolve symlinks in. 2329 * @return Generic type determined by the implementation of next. 2330 * @throws IOException 2331 */ 2332 public T resolve(final FileContext fc, Path p) throws IOException { 2333 int count = 0; 2334 T in = null; 2335 Path first = p; 2336 // NB: More than one AbstractFileSystem can match a scheme, eg 2337 // "file" resolves to LocalFs but could have come by RawLocalFs. 2338 AbstractFileSystem fs = fc.getFSofPath(p); 2339 2340 // Loop until all symlinks are resolved or the limit is reached 2341 for (boolean isLink = true; isLink;) { 2342 try { 2343 in = next(fs, p); 2344 isLink = false; 2345 } catch (UnresolvedLinkException e) { 2346 if (count++ > MAX_PATH_LINKS) { 2347 throw new IOException("Possible cyclic loop while " + 2348 "following symbolic link " + first); 2349 } 2350 // Resolve the first unresolved path component 2351 p = qualifySymlinkTarget(fs, p, fs.getLinkTarget(p)); 2352 fs = fc.getFSofPath(p); 2353 } 2354 } 2355 return in; 2356 } 2357 } 2358 2359 /** 2360 * Get the statistics for a particular file system 2361 * 2362 * @param uri 2363 * the uri to lookup the statistics. Only scheme and authority part 2364 * of the uri are used as the key to store and lookup. 2365 * @return a statistics object 2366 */ 2367 public static Statistics getStatistics(URI uri) { 2368 return AbstractFileSystem.getStatistics(uri); 2369 } 2370 2371 /** 2372 * Clears all the statistics stored in AbstractFileSystem, for all the file 2373 * systems. 2374 */ 2375 public static void clearStatistics() { 2376 AbstractFileSystem.clearStatistics(); 2377 } 2378 2379 /** 2380 * Prints the statistics to standard output. File System is identified by the 2381 * scheme and authority. 2382 */ 2383 public static void printStatistics() { 2384 AbstractFileSystem.printStatistics(); 2385 } 2386 2387 /** 2388 * @return Map of uri and statistics for each filesystem instantiated. The uri 2389 * consists of scheme and authority for the filesystem. 2390 */ 2391 public static Map<URI, Statistics> getAllStatistics() { 2392 return AbstractFileSystem.getAllStatistics(); 2393 } 2394 2395 /** 2396 * Get delegation tokens for the file systems accessed for a given 2397 * path. 2398 * @param p Path for which delegations tokens are requested. 2399 * @param renewer the account name that is allowed to renew the token. 2400 * @return List of delegation tokens. 2401 * @throws IOException 2402 */ 2403 @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" }) 2404 public List<Token<?>> getDelegationTokens( 2405 Path p, String renewer) throws IOException { 2406 Set<AbstractFileSystem> afsSet = resolveAbstractFileSystems(p); 2407 List<Token<?>> tokenList = 2408 new ArrayList<Token<?>>(); 2409 for (AbstractFileSystem afs : afsSet) { 2410 List<Token<?>> afsTokens = afs.getDelegationTokens(renewer); 2411 tokenList.addAll(afsTokens); 2412 } 2413 return tokenList; 2414 } 2415 }