001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs; 019 020 import java.io.Closeable; 021 import java.io.FileNotFoundException; 022 import java.io.IOException; 023 import java.lang.ref.WeakReference; 024 import java.net.URI; 025 import java.net.URISyntaxException; 026 import java.security.PrivilegedExceptionAction; 027 import java.util.ArrayList; 028 import java.util.Arrays; 029 import java.util.Collections; 030 import java.util.EnumSet; 031 import java.util.HashMap; 032 import java.util.HashSet; 033 import java.util.IdentityHashMap; 034 import java.util.Iterator; 035 import java.util.LinkedList; 036 import java.util.List; 037 import java.util.Map; 038 import java.util.NoSuchElementException; 039 import java.util.ServiceLoader; 040 import java.util.Set; 041 import java.util.Stack; 042 import java.util.TreeSet; 043 import java.util.concurrent.atomic.AtomicInteger; 044 import java.util.concurrent.atomic.AtomicLong; 045 046 import org.apache.commons.logging.Log; 047 import org.apache.commons.logging.LogFactory; 048 import org.apache.hadoop.classification.InterfaceAudience; 049 import org.apache.hadoop.classification.InterfaceStability; 050 import org.apache.hadoop.conf.Configuration; 051 import org.apache.hadoop.conf.Configured; 052 import org.apache.hadoop.fs.Options.ChecksumOpt; 053 import org.apache.hadoop.fs.Options.Rename; 054 import org.apache.hadoop.fs.permission.FsPermission; 055 import org.apache.hadoop.io.MultipleIOException; 056 import org.apache.hadoop.io.Text; 057 import org.apache.hadoop.net.NetUtils; 058 import org.apache.hadoop.security.Credentials; 059 import org.apache.hadoop.security.SecurityUtil; 060 import org.apache.hadoop.security.UserGroupInformation; 061 import org.apache.hadoop.security.token.Token; 062 import org.apache.hadoop.util.DataChecksum; 063 import org.apache.hadoop.util.Progressable; 064 import org.apache.hadoop.util.ReflectionUtils; 065 import org.apache.hadoop.util.ShutdownHookManager; 066 067 import com.google.common.annotations.VisibleForTesting; 068 069 /**************************************************************** 070 * An abstract base class for a fairly generic filesystem. It 071 * may be implemented as a distributed filesystem, or as a "local" 072 * one that reflects the locally-connected disk. The local version 073 * exists for small Hadoop instances and for testing. 074 * 075 * <p> 076 * 077 * All user code that may potentially use the Hadoop Distributed 078 * File System should be written to use a FileSystem object. The 079 * Hadoop DFS is a multi-machine system that appears as a single 080 * disk. It's useful because of its fault tolerance and potentially 081 * very large capacity. 082 * 083 * <p> 084 * The local implementation is {@link LocalFileSystem} and distributed 085 * implementation is DistributedFileSystem. 086 *****************************************************************/ 087 @InterfaceAudience.Public 088 @InterfaceStability.Stable 089 public abstract class FileSystem extends Configured implements Closeable { 090 public static final String FS_DEFAULT_NAME_KEY = 091 CommonConfigurationKeys.FS_DEFAULT_NAME_KEY; 092 public static final String DEFAULT_FS = 093 CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT; 094 095 public static final Log LOG = LogFactory.getLog(FileSystem.class); 096 097 /** 098 * Priority of the FileSystem shutdown hook. 099 */ 100 public static final int SHUTDOWN_HOOK_PRIORITY = 10; 101 102 /** FileSystem cache */ 103 static final Cache CACHE = new Cache(); 104 105 /** The key this instance is stored under in the cache. */ 106 private Cache.Key key; 107 108 /** Recording statistics per a FileSystem class */ 109 private static final Map<Class<? extends FileSystem>, Statistics> 110 statisticsTable = 111 new IdentityHashMap<Class<? extends FileSystem>, Statistics>(); 112 113 /** 114 * The statistics for this file system. 115 */ 116 protected Statistics statistics; 117 118 /** 119 * A cache of files that should be deleted when filsystem is closed 120 * or the JVM is exited. 121 */ 122 private Set<Path> deleteOnExit = new TreeSet<Path>(); 123 124 /** 125 * This method adds a file system for testing so that we can find it later. It 126 * is only for testing. 127 * @param uri the uri to store it under 128 * @param conf the configuration to store it under 129 * @param fs the file system to store 130 * @throws IOException 131 */ 132 static void addFileSystemForTesting(URI uri, Configuration conf, 133 FileSystem fs) throws IOException { 134 CACHE.map.put(new Cache.Key(uri, conf), fs); 135 } 136 137 /** 138 * Get a filesystem instance based on the uri, the passed 139 * configuration and the user 140 * @param uri of the filesystem 141 * @param conf the configuration to use 142 * @param user to perform the get as 143 * @return the filesystem instance 144 * @throws IOException 145 * @throws InterruptedException 146 */ 147 public static FileSystem get(final URI uri, final Configuration conf, 148 final String user) throws IOException, InterruptedException { 149 String ticketCachePath = 150 conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); 151 UserGroupInformation ugi = 152 UserGroupInformation.getBestUGI(ticketCachePath, user); 153 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 154 public FileSystem run() throws IOException { 155 return get(uri, conf); 156 } 157 }); 158 } 159 160 /** 161 * Returns the configured filesystem implementation. 162 * @param conf the configuration to use 163 */ 164 public static FileSystem get(Configuration conf) throws IOException { 165 return get(getDefaultUri(conf), conf); 166 } 167 168 /** Get the default filesystem URI from a configuration. 169 * @param conf the configuration to use 170 * @return the uri of the default filesystem 171 */ 172 public static URI getDefaultUri(Configuration conf) { 173 return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS))); 174 } 175 176 /** Set the default filesystem URI in a configuration. 177 * @param conf the configuration to alter 178 * @param uri the new default filesystem uri 179 */ 180 public static void setDefaultUri(Configuration conf, URI uri) { 181 conf.set(FS_DEFAULT_NAME_KEY, uri.toString()); 182 } 183 184 /** Set the default filesystem URI in a configuration. 185 * @param conf the configuration to alter 186 * @param uri the new default filesystem uri 187 */ 188 public static void setDefaultUri(Configuration conf, String uri) { 189 setDefaultUri(conf, URI.create(fixName(uri))); 190 } 191 192 /** Called after a new FileSystem instance is constructed. 193 * @param name a uri whose authority section names the host, port, etc. 194 * for this FileSystem 195 * @param conf the configuration 196 */ 197 public void initialize(URI name, Configuration conf) throws IOException { 198 statistics = getStatistics(name.getScheme(), getClass()); 199 } 200 201 /** 202 * Return the protocol scheme for the FileSystem. 203 * <p/> 204 * This implementation throws an <code>UnsupportedOperationException</code>. 205 * 206 * @return the protocol scheme for the FileSystem. 207 */ 208 public String getScheme() { 209 throw new UnsupportedOperationException("Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation"); 210 } 211 212 /** Returns a URI whose scheme and authority identify this FileSystem.*/ 213 public abstract URI getUri(); 214 215 /** 216 * Return a canonicalized form of this FileSystem's URI. 217 * 218 * The default implementation simply calls {@link #canonicalizeUri(URI)} 219 * on the filesystem's own URI, so subclasses typically only need to 220 * implement that method. 221 * 222 * @see #canonicalizeUri(URI) 223 */ 224 protected URI getCanonicalUri() { 225 return canonicalizeUri(getUri()); 226 } 227 228 /** 229 * Canonicalize the given URI. 230 * 231 * This is filesystem-dependent, but may for example consist of 232 * canonicalizing the hostname using DNS and adding the default 233 * port if not specified. 234 * 235 * The default implementation simply fills in the default port if 236 * not specified and if the filesystem has a default port. 237 * 238 * @return URI 239 * @see NetUtils#getCanonicalUri(URI, int) 240 */ 241 protected URI canonicalizeUri(URI uri) { 242 if (uri.getPort() == -1 && getDefaultPort() > 0) { 243 // reconstruct the uri with the default port set 244 try { 245 uri = new URI(uri.getScheme(), uri.getUserInfo(), 246 uri.getHost(), getDefaultPort(), 247 uri.getPath(), uri.getQuery(), uri.getFragment()); 248 } catch (URISyntaxException e) { 249 // Should never happen! 250 throw new AssertionError("Valid URI became unparseable: " + 251 uri); 252 } 253 } 254 255 return uri; 256 } 257 258 /** 259 * Get the default port for this file system. 260 * @return the default port or 0 if there isn't one 261 */ 262 protected int getDefaultPort() { 263 return 0; 264 } 265 266 /** 267 * Get a canonical service name for this file system. The token cache is 268 * the only user of the canonical service name, and uses it to lookup this 269 * filesystem's service tokens. 270 * If file system provides a token of its own then it must have a canonical 271 * name, otherwise canonical name can be null. 272 * 273 * Default Impl: If the file system has child file systems 274 * (such as an embedded file system) then it is assumed that the fs has no 275 * tokens of its own and hence returns a null name; otherwise a service 276 * name is built using Uri and port. 277 * 278 * @return a service string that uniquely identifies this file system, null 279 * if the filesystem does not implement tokens 280 * @see SecurityUtil#buildDTServiceName(URI, int) 281 */ 282 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" }) 283 public String getCanonicalServiceName() { 284 return (getChildFileSystems() == null) 285 ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()) 286 : null; 287 } 288 289 /** @deprecated call #getUri() instead.*/ 290 @Deprecated 291 public String getName() { return getUri().toString(); } 292 293 /** @deprecated call #get(URI,Configuration) instead. */ 294 @Deprecated 295 public static FileSystem getNamed(String name, Configuration conf) 296 throws IOException { 297 return get(URI.create(fixName(name)), conf); 298 } 299 300 /** Update old-format filesystem names, for back-compatibility. This should 301 * eventually be replaced with a checkName() method that throws an exception 302 * for old-format names. */ 303 private static String fixName(String name) { 304 // convert old-format name to new-format name 305 if (name.equals("local")) { // "local" is now "file:///". 306 LOG.warn("\"local\" is a deprecated filesystem name." 307 +" Use \"file:///\" instead."); 308 name = "file:///"; 309 } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://" 310 LOG.warn("\""+name+"\" is a deprecated filesystem name." 311 +" Use \"hdfs://"+name+"/\" instead."); 312 name = "hdfs://"+name; 313 } 314 return name; 315 } 316 317 /** 318 * Get the local file system. 319 * @param conf the configuration to configure the file system with 320 * @return a LocalFileSystem 321 */ 322 public static LocalFileSystem getLocal(Configuration conf) 323 throws IOException { 324 return (LocalFileSystem)get(LocalFileSystem.NAME, conf); 325 } 326 327 /** Returns the FileSystem for this URI's scheme and authority. The scheme 328 * of the URI determines a configuration property name, 329 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class. 330 * The entire URI is passed to the FileSystem instance's initialize method. 331 */ 332 public static FileSystem get(URI uri, Configuration conf) throws IOException { 333 String scheme = uri.getScheme(); 334 String authority = uri.getAuthority(); 335 336 if (scheme == null && authority == null) { // use default FS 337 return get(conf); 338 } 339 340 if (scheme != null && authority == null) { // no authority 341 URI defaultUri = getDefaultUri(conf); 342 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default 343 && defaultUri.getAuthority() != null) { // & default has authority 344 return get(defaultUri, conf); // return default 345 } 346 } 347 348 String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); 349 if (conf.getBoolean(disableCacheName, false)) { 350 return createFileSystem(uri, conf); 351 } 352 353 return CACHE.get(uri, conf); 354 } 355 356 /** 357 * Returns the FileSystem for this URI's scheme and authority and the 358 * passed user. Internally invokes {@link #newInstance(URI, Configuration)} 359 * @param uri of the filesystem 360 * @param conf the configuration to use 361 * @param user to perform the get as 362 * @return filesystem instance 363 * @throws IOException 364 * @throws InterruptedException 365 */ 366 public static FileSystem newInstance(final URI uri, final Configuration conf, 367 final String user) throws IOException, InterruptedException { 368 String ticketCachePath = 369 conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); 370 UserGroupInformation ugi = 371 UserGroupInformation.getBestUGI(ticketCachePath, user); 372 return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 373 public FileSystem run() throws IOException { 374 return newInstance(uri,conf); 375 } 376 }); 377 } 378 /** Returns the FileSystem for this URI's scheme and authority. The scheme 379 * of the URI determines a configuration property name, 380 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class. 381 * The entire URI is passed to the FileSystem instance's initialize method. 382 * This always returns a new FileSystem object. 383 */ 384 public static FileSystem newInstance(URI uri, Configuration conf) throws IOException { 385 String scheme = uri.getScheme(); 386 String authority = uri.getAuthority(); 387 388 if (scheme == null) { // no scheme: use default FS 389 return newInstance(conf); 390 } 391 392 if (authority == null) { // no authority 393 URI defaultUri = getDefaultUri(conf); 394 if (scheme.equals(defaultUri.getScheme()) // if scheme matches default 395 && defaultUri.getAuthority() != null) { // & default has authority 396 return newInstance(defaultUri, conf); // return default 397 } 398 } 399 return CACHE.getUnique(uri, conf); 400 } 401 402 /** Returns a unique configured filesystem implementation. 403 * This always returns a new FileSystem object. 404 * @param conf the configuration to use 405 */ 406 public static FileSystem newInstance(Configuration conf) throws IOException { 407 return newInstance(getDefaultUri(conf), conf); 408 } 409 410 /** 411 * Get a unique local file system object 412 * @param conf the configuration to configure the file system with 413 * @return a LocalFileSystem 414 * This always returns a new FileSystem object. 415 */ 416 public static LocalFileSystem newInstanceLocal(Configuration conf) 417 throws IOException { 418 return (LocalFileSystem)newInstance(LocalFileSystem.NAME, conf); 419 } 420 421 /** 422 * Close all cached filesystems. Be sure those filesystems are not 423 * used anymore. 424 * 425 * @throws IOException 426 */ 427 public static void closeAll() throws IOException { 428 CACHE.closeAll(); 429 } 430 431 /** 432 * Close all cached filesystems for a given UGI. Be sure those filesystems 433 * are not used anymore. 434 * @param ugi user group info to close 435 * @throws IOException 436 */ 437 public static void closeAllForUGI(UserGroupInformation ugi) 438 throws IOException { 439 CACHE.closeAll(ugi); 440 } 441 442 /** 443 * Make sure that a path specifies a FileSystem. 444 * @param path to use 445 */ 446 public Path makeQualified(Path path) { 447 checkPath(path); 448 return path.makeQualified(this.getUri(), this.getWorkingDirectory()); 449 } 450 451 /** 452 * Get a new delegation token for this file system. 453 * This is an internal method that should have been declared protected 454 * but wasn't historically. 455 * Callers should use {@link #addDelegationTokens(String, Credentials)} 456 * 457 * @param renewer the account name that is allowed to renew the token. 458 * @return a new delegation token 459 * @throws IOException 460 */ 461 @InterfaceAudience.Private() 462 public Token<?> getDelegationToken(String renewer) throws IOException { 463 return null; 464 } 465 466 /** 467 * Obtain all delegation tokens used by this FileSystem that are not 468 * already present in the given Credentials. Existing tokens will neither 469 * be verified as valid nor having the given renewer. Missing tokens will 470 * be acquired and added to the given Credentials. 471 * 472 * Default Impl: works for simple fs with its own token 473 * and also for an embedded fs whose tokens are those of its 474 * children file system (i.e. the embedded fs has not tokens of its 475 * own). 476 * 477 * @param renewer the user allowed to renew the delegation tokens 478 * @param credentials cache in which to add new delegation tokens 479 * @return list of new delegation tokens 480 * @throws IOException 481 */ 482 @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" }) 483 public Token<?>[] addDelegationTokens( 484 final String renewer, Credentials credentials) throws IOException { 485 if (credentials == null) { 486 credentials = new Credentials(); 487 } 488 final List<Token<?>> tokens = new ArrayList<Token<?>>(); 489 collectDelegationTokens(renewer, credentials, tokens); 490 return tokens.toArray(new Token<?>[tokens.size()]); 491 } 492 493 /** 494 * Recursively obtain the tokens for this FileSystem and all descended 495 * FileSystems as determined by getChildFileSystems(). 496 * @param renewer the user allowed to renew the delegation tokens 497 * @param credentials cache in which to add the new delegation tokens 498 * @param tokens list in which to add acquired tokens 499 * @throws IOException 500 */ 501 private void collectDelegationTokens(final String renewer, 502 final Credentials credentials, 503 final List<Token<?>> tokens) 504 throws IOException { 505 final String serviceName = getCanonicalServiceName(); 506 // Collect token of the this filesystem and then of its embedded children 507 if (serviceName != null) { // fs has token, grab it 508 final Text service = new Text(serviceName); 509 Token<?> token = credentials.getToken(service); 510 if (token == null) { 511 token = getDelegationToken(renewer); 512 if (token != null) { 513 tokens.add(token); 514 credentials.addToken(service, token); 515 } 516 } 517 } 518 // Now collect the tokens from the children 519 final FileSystem[] children = getChildFileSystems(); 520 if (children != null) { 521 for (final FileSystem fs : children) { 522 fs.collectDelegationTokens(renewer, credentials, tokens); 523 } 524 } 525 } 526 527 /** 528 * Get all the immediate child FileSystems embedded in this FileSystem. 529 * It does not recurse and get grand children. If a FileSystem 530 * has multiple child FileSystems, then it should return a unique list 531 * of those FileSystems. Default is to return null to signify no children. 532 * 533 * @return FileSystems used by this FileSystem 534 */ 535 @InterfaceAudience.LimitedPrivate({ "HDFS" }) 536 @VisibleForTesting 537 public FileSystem[] getChildFileSystems() { 538 return null; 539 } 540 541 /** create a file with the provided permission 542 * The permission of the file is set to be the provided permission as in 543 * setPermission, not permission&~umask 544 * 545 * It is implemented using two RPCs. It is understood that it is inefficient, 546 * but the implementation is thread-safe. The other option is to change the 547 * value of umask in configuration to be 0, but it is not thread-safe. 548 * 549 * @param fs file system handle 550 * @param file the name of the file to be created 551 * @param permission the permission of the file 552 * @return an output stream 553 * @throws IOException 554 */ 555 public static FSDataOutputStream create(FileSystem fs, 556 Path file, FsPermission permission) throws IOException { 557 // create the file with default permission 558 FSDataOutputStream out = fs.create(file); 559 // set its permission to the supplied one 560 fs.setPermission(file, permission); 561 return out; 562 } 563 564 /** create a directory with the provided permission 565 * The permission of the directory is set to be the provided permission as in 566 * setPermission, not permission&~umask 567 * 568 * @see #create(FileSystem, Path, FsPermission) 569 * 570 * @param fs file system handle 571 * @param dir the name of the directory to be created 572 * @param permission the permission of the directory 573 * @return true if the directory creation succeeds; false otherwise 574 * @throws IOException 575 */ 576 public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) 577 throws IOException { 578 // create the directory using the default permission 579 boolean result = fs.mkdirs(dir); 580 // set its permission to be the supplied one 581 fs.setPermission(dir, permission); 582 return result; 583 } 584 585 /////////////////////////////////////////////////////////////// 586 // FileSystem 587 /////////////////////////////////////////////////////////////// 588 589 protected FileSystem() { 590 super(null); 591 } 592 593 /** 594 * Check that a Path belongs to this FileSystem. 595 * @param path to check 596 */ 597 protected void checkPath(Path path) { 598 URI uri = path.toUri(); 599 String thatScheme = uri.getScheme(); 600 if (thatScheme == null) // fs is relative 601 return; 602 URI thisUri = getCanonicalUri(); 603 String thisScheme = thisUri.getScheme(); 604 //authority and scheme are not case sensitive 605 if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match 606 String thisAuthority = thisUri.getAuthority(); 607 String thatAuthority = uri.getAuthority(); 608 if (thatAuthority == null && // path's authority is null 609 thisAuthority != null) { // fs has an authority 610 URI defaultUri = getDefaultUri(getConf()); 611 if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) { 612 uri = defaultUri; // schemes match, so use this uri instead 613 } else { 614 uri = null; // can't determine auth of the path 615 } 616 } 617 if (uri != null) { 618 // canonicalize uri before comparing with this fs 619 uri = canonicalizeUri(uri); 620 thatAuthority = uri.getAuthority(); 621 if (thisAuthority == thatAuthority || // authorities match 622 (thisAuthority != null && 623 thisAuthority.equalsIgnoreCase(thatAuthority))) 624 return; 625 } 626 } 627 throw new IllegalArgumentException("Wrong FS: "+path+ 628 ", expected: "+this.getUri()); 629 } 630 631 /** 632 * Return an array containing hostnames, offset and size of 633 * portions of the given file. For a nonexistent 634 * file or regions, null will be returned. 635 * 636 * This call is most helpful with DFS, where it returns 637 * hostnames of machines that contain the given file. 638 * 639 * The FileSystem will simply return an elt containing 'localhost'. 640 * 641 * @param file FilesStatus to get data from 642 * @param start offset into the given file 643 * @param len length for which to get locations for 644 */ 645 public BlockLocation[] getFileBlockLocations(FileStatus file, 646 long start, long len) throws IOException { 647 if (file == null) { 648 return null; 649 } 650 651 if (start < 0 || len < 0) { 652 throw new IllegalArgumentException("Invalid start or len parameter"); 653 } 654 655 if (file.getLen() <= start) { 656 return new BlockLocation[0]; 657 658 } 659 String[] name = { "localhost:50010" }; 660 String[] host = { "localhost" }; 661 return new BlockLocation[] { 662 new BlockLocation(name, host, 0, file.getLen()) }; 663 } 664 665 666 /** 667 * Return an array containing hostnames, offset and size of 668 * portions of the given file. For a nonexistent 669 * file or regions, null will be returned. 670 * 671 * This call is most helpful with DFS, where it returns 672 * hostnames of machines that contain the given file. 673 * 674 * The FileSystem will simply return an elt containing 'localhost'. 675 * 676 * @param p path is used to identify an FS since an FS could have 677 * another FS that it could be delegating the call to 678 * @param start offset into the given file 679 * @param len length for which to get locations for 680 */ 681 public BlockLocation[] getFileBlockLocations(Path p, 682 long start, long len) throws IOException { 683 if (p == null) { 684 throw new NullPointerException(); 685 } 686 FileStatus file = getFileStatus(p); 687 return getFileBlockLocations(file, start, len); 688 } 689 690 /** 691 * Return a set of server default configuration values 692 * @return server default configuration values 693 * @throws IOException 694 * @deprecated use {@link #getServerDefaults(Path)} instead 695 */ 696 @Deprecated 697 public FsServerDefaults getServerDefaults() throws IOException { 698 Configuration conf = getConf(); 699 // CRC32 is chosen as default as it is available in all 700 // releases that support checksum. 701 // The client trash configuration is ignored. 702 return new FsServerDefaults(getDefaultBlockSize(), 703 conf.getInt("io.bytes.per.checksum", 512), 704 64 * 1024, 705 getDefaultReplication(), 706 conf.getInt("io.file.buffer.size", 4096), 707 false, 708 CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT, 709 DataChecksum.CHECKSUM_CRC32); 710 } 711 712 /** 713 * Return a set of server default configuration values 714 * @param p path is used to identify an FS since an FS could have 715 * another FS that it could be delegating the call to 716 * @return server default configuration values 717 * @throws IOException 718 */ 719 public FsServerDefaults getServerDefaults(Path p) throws IOException { 720 return getServerDefaults(); 721 } 722 723 /** 724 * Return the fully-qualified path of path f resolving the path 725 * through any symlinks or mount point 726 * @param p path to be resolved 727 * @return fully qualified path 728 * @throws FileNotFoundException 729 */ 730 public Path resolvePath(final Path p) throws IOException { 731 checkPath(p); 732 return getFileStatus(p).getPath(); 733 } 734 735 /** 736 * Opens an FSDataInputStream at the indicated Path. 737 * @param f the file name to open 738 * @param bufferSize the size of the buffer to be used. 739 */ 740 public abstract FSDataInputStream open(Path f, int bufferSize) 741 throws IOException; 742 743 /** 744 * Opens an FSDataInputStream at the indicated Path. 745 * @param f the file to open 746 */ 747 public FSDataInputStream open(Path f) throws IOException { 748 return open(f, getConf().getInt("io.file.buffer.size", 4096)); 749 } 750 751 /** 752 * Create an FSDataOutputStream at the indicated Path. 753 * Files are overwritten by default. 754 * @param f the file to create 755 */ 756 public FSDataOutputStream create(Path f) throws IOException { 757 return create(f, true); 758 } 759 760 /** 761 * Create an FSDataOutputStream at the indicated Path. 762 * @param f the file to create 763 * @param overwrite if a file with this name already exists, then if true, 764 * the file will be overwritten, and if false an exception will be thrown. 765 */ 766 public FSDataOutputStream create(Path f, boolean overwrite) 767 throws IOException { 768 return create(f, overwrite, 769 getConf().getInt("io.file.buffer.size", 4096), 770 getDefaultReplication(f), 771 getDefaultBlockSize(f)); 772 } 773 774 /** 775 * Create an FSDataOutputStream at the indicated Path with write-progress 776 * reporting. 777 * Files are overwritten by default. 778 * @param f the file to create 779 * @param progress to report progress 780 */ 781 public FSDataOutputStream create(Path f, Progressable progress) 782 throws IOException { 783 return create(f, true, 784 getConf().getInt("io.file.buffer.size", 4096), 785 getDefaultReplication(f), 786 getDefaultBlockSize(f), progress); 787 } 788 789 /** 790 * Create an FSDataOutputStream at the indicated Path. 791 * Files are overwritten by default. 792 * @param f the file to create 793 * @param replication the replication factor 794 */ 795 public FSDataOutputStream create(Path f, short replication) 796 throws IOException { 797 return create(f, true, 798 getConf().getInt("io.file.buffer.size", 4096), 799 replication, 800 getDefaultBlockSize(f)); 801 } 802 803 /** 804 * Create an FSDataOutputStream at the indicated Path with write-progress 805 * reporting. 806 * Files are overwritten by default. 807 * @param f the file to create 808 * @param replication the replication factor 809 * @param progress to report progress 810 */ 811 public FSDataOutputStream create(Path f, short replication, 812 Progressable progress) throws IOException { 813 return create(f, true, 814 getConf().getInt("io.file.buffer.size", 4096), 815 replication, 816 getDefaultBlockSize(f), progress); 817 } 818 819 820 /** 821 * Create an FSDataOutputStream at the indicated Path. 822 * @param f the file name to create 823 * @param overwrite if a file with this name already exists, then if true, 824 * the file will be overwritten, and if false an error will be thrown. 825 * @param bufferSize the size of the buffer to be used. 826 */ 827 public FSDataOutputStream create(Path f, 828 boolean overwrite, 829 int bufferSize 830 ) throws IOException { 831 return create(f, overwrite, bufferSize, 832 getDefaultReplication(f), 833 getDefaultBlockSize(f)); 834 } 835 836 /** 837 * Create an FSDataOutputStream at the indicated Path with write-progress 838 * reporting. 839 * @param f the path of the file to open 840 * @param overwrite if a file with this name already exists, then if true, 841 * the file will be overwritten, and if false an error will be thrown. 842 * @param bufferSize the size of the buffer to be used. 843 */ 844 public FSDataOutputStream create(Path f, 845 boolean overwrite, 846 int bufferSize, 847 Progressable progress 848 ) throws IOException { 849 return create(f, overwrite, bufferSize, 850 getDefaultReplication(f), 851 getDefaultBlockSize(f), progress); 852 } 853 854 855 /** 856 * Create an FSDataOutputStream at the indicated Path. 857 * @param f the file name to open 858 * @param overwrite if a file with this name already exists, then if true, 859 * the file will be overwritten, and if false an error will be thrown. 860 * @param bufferSize the size of the buffer to be used. 861 * @param replication required block replication for the file. 862 */ 863 public FSDataOutputStream create(Path f, 864 boolean overwrite, 865 int bufferSize, 866 short replication, 867 long blockSize 868 ) throws IOException { 869 return create(f, overwrite, bufferSize, replication, blockSize, null); 870 } 871 872 /** 873 * Create an FSDataOutputStream at the indicated Path with write-progress 874 * reporting. 875 * @param f the file name to open 876 * @param overwrite if a file with this name already exists, then if true, 877 * the file will be overwritten, and if false an error will be thrown. 878 * @param bufferSize the size of the buffer to be used. 879 * @param replication required block replication for the file. 880 */ 881 public FSDataOutputStream create(Path f, 882 boolean overwrite, 883 int bufferSize, 884 short replication, 885 long blockSize, 886 Progressable progress 887 ) throws IOException { 888 return this.create(f, FsPermission.getFileDefault().applyUMask( 889 FsPermission.getUMask(getConf())), overwrite, bufferSize, 890 replication, blockSize, progress); 891 } 892 893 /** 894 * Create an FSDataOutputStream at the indicated Path with write-progress 895 * reporting. 896 * @param f the file name to open 897 * @param permission 898 * @param overwrite if a file with this name already exists, then if true, 899 * the file will be overwritten, and if false an error will be thrown. 900 * @param bufferSize the size of the buffer to be used. 901 * @param replication required block replication for the file. 902 * @param blockSize 903 * @param progress 904 * @throws IOException 905 * @see #setPermission(Path, FsPermission) 906 */ 907 public abstract FSDataOutputStream create(Path f, 908 FsPermission permission, 909 boolean overwrite, 910 int bufferSize, 911 short replication, 912 long blockSize, 913 Progressable progress) throws IOException; 914 915 /** 916 * Create an FSDataOutputStream at the indicated Path with write-progress 917 * reporting. 918 * @param f the file name to open 919 * @param permission 920 * @param flags {@link CreateFlag}s to use for this stream. 921 * @param bufferSize the size of the buffer to be used. 922 * @param replication required block replication for the file. 923 * @param blockSize 924 * @param progress 925 * @throws IOException 926 * @see #setPermission(Path, FsPermission) 927 */ 928 public FSDataOutputStream create(Path f, 929 FsPermission permission, 930 EnumSet<CreateFlag> flags, 931 int bufferSize, 932 short replication, 933 long blockSize, 934 Progressable progress) throws IOException { 935 return create(f, permission, flags, bufferSize, replication, 936 blockSize, progress, null); 937 } 938 939 /** 940 * Create an FSDataOutputStream at the indicated Path with a custom 941 * checksum option 942 * @param f the file name to open 943 * @param permission 944 * @param flags {@link CreateFlag}s to use for this stream. 945 * @param bufferSize the size of the buffer to be used. 946 * @param replication required block replication for the file. 947 * @param blockSize 948 * @param progress 949 * @param checksumOpt checksum parameter. If null, the values 950 * found in conf will be used. 951 * @throws IOException 952 * @see #setPermission(Path, FsPermission) 953 */ 954 public FSDataOutputStream create(Path f, 955 FsPermission permission, 956 EnumSet<CreateFlag> flags, 957 int bufferSize, 958 short replication, 959 long blockSize, 960 Progressable progress, 961 ChecksumOpt checksumOpt) throws IOException { 962 // Checksum options are ignored by default. The file systems that 963 // implement checksum need to override this method. The full 964 // support is currently only available in DFS. 965 return create(f, permission, flags.contains(CreateFlag.OVERWRITE), 966 bufferSize, replication, blockSize, progress); 967 } 968 969 /*. 970 * This create has been added to support the FileContext that processes 971 * the permission 972 * with umask before calling this method. 973 * This a temporary method added to support the transition from FileSystem 974 * to FileContext for user applications. 975 */ 976 @Deprecated 977 protected FSDataOutputStream primitiveCreate(Path f, 978 FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize, 979 short replication, long blockSize, Progressable progress, 980 ChecksumOpt checksumOpt) throws IOException { 981 982 boolean pathExists = exists(f); 983 CreateFlag.validate(f, pathExists, flag); 984 985 // Default impl assumes that permissions do not matter and 986 // nor does the bytesPerChecksum hence 987 // calling the regular create is good enough. 988 // FSs that implement permissions should override this. 989 990 if (pathExists && flag.contains(CreateFlag.APPEND)) { 991 return append(f, bufferSize, progress); 992 } 993 994 return this.create(f, absolutePermission, 995 flag.contains(CreateFlag.OVERWRITE), bufferSize, replication, 996 blockSize, progress); 997 } 998 999 /** 1000 * This version of the mkdirs method assumes that the permission is absolute. 1001 * It has been added to support the FileContext that processes the permission 1002 * with umask before calling this method. 1003 * This a temporary method added to support the transition from FileSystem 1004 * to FileContext for user applications. 1005 */ 1006 @Deprecated 1007 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) 1008 throws IOException { 1009 // Default impl is to assume that permissions do not matter and hence 1010 // calling the regular mkdirs is good enough. 1011 // FSs that implement permissions should override this. 1012 return this.mkdirs(f, absolutePermission); 1013 } 1014 1015 1016 /** 1017 * This version of the mkdirs method assumes that the permission is absolute. 1018 * It has been added to support the FileContext that processes the permission 1019 * with umask before calling this method. 1020 * This a temporary method added to support the transition from FileSystem 1021 * to FileContext for user applications. 1022 */ 1023 @Deprecated 1024 protected void primitiveMkdir(Path f, FsPermission absolutePermission, 1025 boolean createParent) 1026 throws IOException { 1027 1028 if (!createParent) { // parent must exist. 1029 // since the this.mkdirs makes parent dirs automatically 1030 // we must throw exception if parent does not exist. 1031 final FileStatus stat = getFileStatus(f.getParent()); 1032 if (stat == null) { 1033 throw new FileNotFoundException("Missing parent:" + f); 1034 } 1035 if (!stat.isDirectory()) { 1036 throw new ParentNotDirectoryException("parent is not a dir"); 1037 } 1038 // parent does exist - go ahead with mkdir of leaf 1039 } 1040 // Default impl is to assume that permissions do not matter and hence 1041 // calling the regular mkdirs is good enough. 1042 // FSs that implement permissions should override this. 1043 if (!this.mkdirs(f, absolutePermission)) { 1044 throw new IOException("mkdir of "+ f + " failed"); 1045 } 1046 } 1047 1048 /** 1049 * Opens an FSDataOutputStream at the indicated Path with write-progress 1050 * reporting. Same as create(), except fails if parent directory doesn't 1051 * already exist. 1052 * @param f the file name to open 1053 * @param overwrite if a file with this name already exists, then if true, 1054 * the file will be overwritten, and if false an error will be thrown. 1055 * @param bufferSize the size of the buffer to be used. 1056 * @param replication required block replication for the file. 1057 * @param blockSize 1058 * @param progress 1059 * @throws IOException 1060 * @see #setPermission(Path, FsPermission) 1061 * @deprecated API only for 0.20-append 1062 */ 1063 @Deprecated 1064 public FSDataOutputStream createNonRecursive(Path f, 1065 boolean overwrite, 1066 int bufferSize, short replication, long blockSize, 1067 Progressable progress) throws IOException { 1068 return this.createNonRecursive(f, FsPermission.getFileDefault(), 1069 overwrite, bufferSize, replication, blockSize, progress); 1070 } 1071 1072 /** 1073 * Opens an FSDataOutputStream at the indicated Path with write-progress 1074 * reporting. Same as create(), except fails if parent directory doesn't 1075 * already exist. 1076 * @param f the file name to open 1077 * @param permission 1078 * @param overwrite if a file with this name already exists, then if true, 1079 * the file will be overwritten, and if false an error will be thrown. 1080 * @param bufferSize the size of the buffer to be used. 1081 * @param replication required block replication for the file. 1082 * @param blockSize 1083 * @param progress 1084 * @throws IOException 1085 * @see #setPermission(Path, FsPermission) 1086 * @deprecated API only for 0.20-append 1087 */ 1088 @Deprecated 1089 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, 1090 boolean overwrite, int bufferSize, short replication, long blockSize, 1091 Progressable progress) throws IOException { 1092 return createNonRecursive(f, permission, 1093 overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) 1094 : EnumSet.of(CreateFlag.CREATE), bufferSize, 1095 replication, blockSize, progress); 1096 } 1097 1098 /** 1099 * Opens an FSDataOutputStream at the indicated Path with write-progress 1100 * reporting. Same as create(), except fails if parent directory doesn't 1101 * already exist. 1102 * @param f the file name to open 1103 * @param permission 1104 * @param flags {@link CreateFlag}s to use for this stream. 1105 * @param bufferSize the size of the buffer to be used. 1106 * @param replication required block replication for the file. 1107 * @param blockSize 1108 * @param progress 1109 * @throws IOException 1110 * @see #setPermission(Path, FsPermission) 1111 * @deprecated API only for 0.20-append 1112 */ 1113 @Deprecated 1114 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, 1115 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, 1116 Progressable progress) throws IOException { 1117 throw new IOException("createNonRecursive unsupported for this filesystem " 1118 + this.getClass()); 1119 } 1120 1121 /** 1122 * Creates the given Path as a brand-new zero-length file. If 1123 * create fails, or if it already existed, return false. 1124 * 1125 * @param f path to use for create 1126 */ 1127 public boolean createNewFile(Path f) throws IOException { 1128 if (exists(f)) { 1129 return false; 1130 } else { 1131 create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close(); 1132 return true; 1133 } 1134 } 1135 1136 /** 1137 * Append to an existing file (optional operation). 1138 * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null) 1139 * @param f the existing file to be appended. 1140 * @throws IOException 1141 */ 1142 public FSDataOutputStream append(Path f) throws IOException { 1143 return append(f, getConf().getInt("io.file.buffer.size", 4096), null); 1144 } 1145 /** 1146 * Append to an existing file (optional operation). 1147 * Same as append(f, bufferSize, null). 1148 * @param f the existing file to be appended. 1149 * @param bufferSize the size of the buffer to be used. 1150 * @throws IOException 1151 */ 1152 public FSDataOutputStream append(Path f, int bufferSize) throws IOException { 1153 return append(f, bufferSize, null); 1154 } 1155 1156 /** 1157 * Append to an existing file (optional operation). 1158 * @param f the existing file to be appended. 1159 * @param bufferSize the size of the buffer to be used. 1160 * @param progress for reporting progress if it is not null. 1161 * @throws IOException 1162 */ 1163 public abstract FSDataOutputStream append(Path f, int bufferSize, 1164 Progressable progress) throws IOException; 1165 1166 /** 1167 * Concat existing files together. 1168 * @param trg the path to the target destination. 1169 * @param psrcs the paths to the sources to use for the concatenation. 1170 * @throws IOException 1171 */ 1172 public void concat(final Path trg, final Path [] psrcs) throws IOException { 1173 throw new UnsupportedOperationException("Not implemented by the " + 1174 getClass().getSimpleName() + " FileSystem implementation"); 1175 } 1176 1177 /** 1178 * Get replication. 1179 * 1180 * @deprecated Use getFileStatus() instead 1181 * @param src file name 1182 * @return file replication 1183 * @throws IOException 1184 */ 1185 @Deprecated 1186 public short getReplication(Path src) throws IOException { 1187 return getFileStatus(src).getReplication(); 1188 } 1189 1190 /** 1191 * Set replication for an existing file. 1192 * 1193 * @param src file name 1194 * @param replication new replication 1195 * @throws IOException 1196 * @return true if successful; 1197 * false if file does not exist or is a directory 1198 */ 1199 public boolean setReplication(Path src, short replication) 1200 throws IOException { 1201 return true; 1202 } 1203 1204 /** 1205 * Renames Path src to Path dst. Can take place on local fs 1206 * or remote DFS. 1207 * @param src path to be renamed 1208 * @param dst new path after rename 1209 * @throws IOException on failure 1210 * @return true if rename is successful 1211 */ 1212 public abstract boolean rename(Path src, Path dst) throws IOException; 1213 1214 /** 1215 * Renames Path src to Path dst 1216 * <ul> 1217 * <li 1218 * <li>Fails if src is a file and dst is a directory. 1219 * <li>Fails if src is a directory and dst is a file. 1220 * <li>Fails if the parent of dst does not exist or is a file. 1221 * </ul> 1222 * <p> 1223 * If OVERWRITE option is not passed as an argument, rename fails 1224 * if the dst already exists. 1225 * <p> 1226 * If OVERWRITE option is passed as an argument, rename overwrites 1227 * the dst if it is a file or an empty directory. Rename fails if dst is 1228 * a non-empty directory. 1229 * <p> 1230 * Note that atomicity of rename is dependent on the file system 1231 * implementation. Please refer to the file system documentation for 1232 * details. This default implementation is non atomic. 1233 * <p> 1234 * This method is deprecated since it is a temporary method added to 1235 * support the transition from FileSystem to FileContext for user 1236 * applications. 1237 * 1238 * @param src path to be renamed 1239 * @param dst new path after rename 1240 * @throws IOException on failure 1241 */ 1242 @Deprecated 1243 protected void rename(final Path src, final Path dst, 1244 final Rename... options) throws IOException { 1245 // Default implementation 1246 final FileStatus srcStatus = getFileStatus(src); 1247 if (srcStatus == null) { 1248 throw new FileNotFoundException("rename source " + src + " not found."); 1249 } 1250 1251 boolean overwrite = false; 1252 if (null != options) { 1253 for (Rename option : options) { 1254 if (option == Rename.OVERWRITE) { 1255 overwrite = true; 1256 } 1257 } 1258 } 1259 1260 FileStatus dstStatus; 1261 try { 1262 dstStatus = getFileStatus(dst); 1263 } catch (IOException e) { 1264 dstStatus = null; 1265 } 1266 if (dstStatus != null) { 1267 if (srcStatus.isDirectory() != dstStatus.isDirectory()) { 1268 throw new IOException("Source " + src + " Destination " + dst 1269 + " both should be either file or directory"); 1270 } 1271 if (!overwrite) { 1272 throw new FileAlreadyExistsException("rename destination " + dst 1273 + " already exists."); 1274 } 1275 // Delete the destination that is a file or an empty directory 1276 if (dstStatus.isDirectory()) { 1277 FileStatus[] list = listStatus(dst); 1278 if (list != null && list.length != 0) { 1279 throw new IOException( 1280 "rename cannot overwrite non empty destination directory " + dst); 1281 } 1282 } 1283 delete(dst, false); 1284 } else { 1285 final Path parent = dst.getParent(); 1286 final FileStatus parentStatus = getFileStatus(parent); 1287 if (parentStatus == null) { 1288 throw new FileNotFoundException("rename destination parent " + parent 1289 + " not found."); 1290 } 1291 if (!parentStatus.isDirectory()) { 1292 throw new ParentNotDirectoryException("rename destination parent " + parent 1293 + " is a file."); 1294 } 1295 } 1296 if (!rename(src, dst)) { 1297 throw new IOException("rename from " + src + " to " + dst + " failed."); 1298 } 1299 } 1300 1301 /** 1302 * Delete a file 1303 * @deprecated Use {@link #delete(Path, boolean)} instead. 1304 */ 1305 @Deprecated 1306 public boolean delete(Path f) throws IOException { 1307 return delete(f, true); 1308 } 1309 1310 /** Delete a file. 1311 * 1312 * @param f the path to delete. 1313 * @param recursive if path is a directory and set to 1314 * true, the directory is deleted else throws an exception. In 1315 * case of a file the recursive can be set to either true or false. 1316 * @return true if delete is successful else false. 1317 * @throws IOException 1318 */ 1319 public abstract boolean delete(Path f, boolean recursive) throws IOException; 1320 1321 /** 1322 * Mark a path to be deleted when FileSystem is closed. 1323 * When the JVM shuts down, 1324 * all FileSystem objects will be closed automatically. 1325 * Then, 1326 * the marked path will be deleted as a result of closing the FileSystem. 1327 * 1328 * The path has to exist in the file system. 1329 * 1330 * @param f the path to delete. 1331 * @return true if deleteOnExit is successful, otherwise false. 1332 * @throws IOException 1333 */ 1334 public boolean deleteOnExit(Path f) throws IOException { 1335 if (!exists(f)) { 1336 return false; 1337 } 1338 synchronized (deleteOnExit) { 1339 deleteOnExit.add(f); 1340 } 1341 return true; 1342 } 1343 1344 /** 1345 * Cancel the deletion of the path when the FileSystem is closed 1346 * @param f the path to cancel deletion 1347 */ 1348 public boolean cancelDeleteOnExit(Path f) { 1349 synchronized (deleteOnExit) { 1350 return deleteOnExit.remove(f); 1351 } 1352 } 1353 1354 /** 1355 * Delete all files that were marked as delete-on-exit. This recursively 1356 * deletes all files in the specified paths. 1357 */ 1358 protected void processDeleteOnExit() { 1359 synchronized (deleteOnExit) { 1360 for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) { 1361 Path path = iter.next(); 1362 try { 1363 if (exists(path)) { 1364 delete(path, true); 1365 } 1366 } 1367 catch (IOException e) { 1368 LOG.info("Ignoring failure to deleteOnExit for path " + path); 1369 } 1370 iter.remove(); 1371 } 1372 } 1373 } 1374 1375 /** Check if exists. 1376 * @param f source file 1377 */ 1378 public boolean exists(Path f) throws IOException { 1379 try { 1380 return getFileStatus(f) != null; 1381 } catch (FileNotFoundException e) { 1382 return false; 1383 } 1384 } 1385 1386 /** True iff the named path is a directory. 1387 * Note: Avoid using this method. Instead reuse the FileStatus 1388 * returned by getFileStatus() or listStatus() methods. 1389 * @param f path to check 1390 */ 1391 public boolean isDirectory(Path f) throws IOException { 1392 try { 1393 return getFileStatus(f).isDirectory(); 1394 } catch (FileNotFoundException e) { 1395 return false; // f does not exist 1396 } 1397 } 1398 1399 /** True iff the named path is a regular file. 1400 * Note: Avoid using this method. Instead reuse the FileStatus 1401 * returned by getFileStatus() or listStatus() methods. 1402 * @param f path to check 1403 */ 1404 public boolean isFile(Path f) throws IOException { 1405 try { 1406 return getFileStatus(f).isFile(); 1407 } catch (FileNotFoundException e) { 1408 return false; // f does not exist 1409 } 1410 } 1411 1412 /** The number of bytes in a file. */ 1413 /** @deprecated Use getFileStatus() instead */ 1414 @Deprecated 1415 public long getLength(Path f) throws IOException { 1416 return getFileStatus(f).getLen(); 1417 } 1418 1419 /** Return the {@link ContentSummary} of a given {@link Path}. 1420 * @param f path to use 1421 */ 1422 public ContentSummary getContentSummary(Path f) throws IOException { 1423 FileStatus status = getFileStatus(f); 1424 if (status.isFile()) { 1425 // f is a file 1426 return new ContentSummary(status.getLen(), 1, 0); 1427 } 1428 // f is a directory 1429 long[] summary = {0, 0, 1}; 1430 for(FileStatus s : listStatus(f)) { 1431 ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : 1432 new ContentSummary(s.getLen(), 1, 0); 1433 summary[0] += c.getLength(); 1434 summary[1] += c.getFileCount(); 1435 summary[2] += c.getDirectoryCount(); 1436 } 1437 return new ContentSummary(summary[0], summary[1], summary[2]); 1438 } 1439 1440 final private static PathFilter DEFAULT_FILTER = new PathFilter() { 1441 public boolean accept(Path file) { 1442 return true; 1443 } 1444 }; 1445 1446 /** 1447 * List the statuses of the files/directories in the given path if the path is 1448 * a directory. 1449 * 1450 * @param f given path 1451 * @return the statuses of the files/directories in the given patch 1452 * @throws FileNotFoundException when the path does not exist; 1453 * IOException see specific implementation 1454 */ 1455 public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, 1456 IOException; 1457 1458 /* 1459 * Filter files/directories in the given path using the user-supplied path 1460 * filter. Results are added to the given array <code>results</code>. 1461 */ 1462 private void listStatus(ArrayList<FileStatus> results, Path f, 1463 PathFilter filter) throws FileNotFoundException, IOException { 1464 FileStatus listing[] = listStatus(f); 1465 if (listing == null) { 1466 throw new IOException("Error accessing " + f); 1467 } 1468 1469 for (int i = 0; i < listing.length; i++) { 1470 if (filter.accept(listing[i].getPath())) { 1471 results.add(listing[i]); 1472 } 1473 } 1474 } 1475 1476 /** 1477 * @return an iterator over the corrupt files under the given path 1478 * (may contain duplicates if a file has more than one corrupt block) 1479 * @throws IOException 1480 */ 1481 public RemoteIterator<Path> listCorruptFileBlocks(Path path) 1482 throws IOException { 1483 throw new UnsupportedOperationException(getClass().getCanonicalName() + 1484 " does not support" + 1485 " listCorruptFileBlocks"); 1486 } 1487 1488 /** 1489 * Filter files/directories in the given path using the user-supplied path 1490 * filter. 1491 * 1492 * @param f 1493 * a path name 1494 * @param filter 1495 * the user-supplied path filter 1496 * @return an array of FileStatus objects for the files under the given path 1497 * after applying the filter 1498 * @throws FileNotFoundException when the path does not exist; 1499 * IOException see specific implementation 1500 */ 1501 public FileStatus[] listStatus(Path f, PathFilter filter) 1502 throws FileNotFoundException, IOException { 1503 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1504 listStatus(results, f, filter); 1505 return results.toArray(new FileStatus[results.size()]); 1506 } 1507 1508 /** 1509 * Filter files/directories in the given list of paths using default 1510 * path filter. 1511 * 1512 * @param files 1513 * a list of paths 1514 * @return a list of statuses for the files under the given paths after 1515 * applying the filter default Path filter 1516 * @throws FileNotFoundException when the path does not exist; 1517 * IOException see specific implementation 1518 */ 1519 public FileStatus[] listStatus(Path[] files) 1520 throws FileNotFoundException, IOException { 1521 return listStatus(files, DEFAULT_FILTER); 1522 } 1523 1524 /** 1525 * Filter files/directories in the given list of paths using user-supplied 1526 * path filter. 1527 * 1528 * @param files 1529 * a list of paths 1530 * @param filter 1531 * the user-supplied path filter 1532 * @return a list of statuses for the files under the given paths after 1533 * applying the filter 1534 * @throws FileNotFoundException when the path does not exist; 1535 * IOException see specific implementation 1536 */ 1537 public FileStatus[] listStatus(Path[] files, PathFilter filter) 1538 throws FileNotFoundException, IOException { 1539 ArrayList<FileStatus> results = new ArrayList<FileStatus>(); 1540 for (int i = 0; i < files.length; i++) { 1541 listStatus(results, files[i], filter); 1542 } 1543 return results.toArray(new FileStatus[results.size()]); 1544 } 1545 1546 /** 1547 * <p>Return all the files that match filePattern and are not checksum 1548 * files. Results are sorted by their names. 1549 * 1550 * <p> 1551 * A filename pattern is composed of <i>regular</i> characters and 1552 * <i>special pattern matching</i> characters, which are: 1553 * 1554 * <dl> 1555 * <dd> 1556 * <dl> 1557 * <p> 1558 * <dt> <tt> ? </tt> 1559 * <dd> Matches any single character. 1560 * 1561 * <p> 1562 * <dt> <tt> * </tt> 1563 * <dd> Matches zero or more characters. 1564 * 1565 * <p> 1566 * <dt> <tt> [<i>abc</i>] </tt> 1567 * <dd> Matches a single character from character set 1568 * <tt>{<i>a,b,c</i>}</tt>. 1569 * 1570 * <p> 1571 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> 1572 * <dd> Matches a single character from the character range 1573 * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be 1574 * lexicographically less than or equal to character <tt><i>b</i></tt>. 1575 * 1576 * <p> 1577 * <dt> <tt> [^<i>a</i>] </tt> 1578 * <dd> Matches a single character that is not from character set or range 1579 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur 1580 * immediately to the right of the opening bracket. 1581 * 1582 * <p> 1583 * <dt> <tt> \<i>c</i> </tt> 1584 * <dd> Removes (escapes) any special meaning of character <i>c</i>. 1585 * 1586 * <p> 1587 * <dt> <tt> {ab,cd} </tt> 1588 * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> 1589 * 1590 * <p> 1591 * <dt> <tt> {ab,c{de,fh}} </tt> 1592 * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt> 1593 * 1594 * </dl> 1595 * </dd> 1596 * </dl> 1597 * 1598 * @param pathPattern a regular expression specifying a pth pattern 1599 1600 * @return an array of paths that match the path pattern 1601 * @throws IOException 1602 */ 1603 public FileStatus[] globStatus(Path pathPattern) throws IOException { 1604 return globStatus(pathPattern, DEFAULT_FILTER); 1605 } 1606 1607 /** 1608 * Return an array of FileStatus objects whose path names match pathPattern 1609 * and is accepted by the user-supplied path filter. Results are sorted by 1610 * their path names. 1611 * Return null if pathPattern has no glob and the path does not exist. 1612 * Return an empty array if pathPattern has a glob and no path matches it. 1613 * 1614 * @param pathPattern 1615 * a regular expression specifying the path pattern 1616 * @param filter 1617 * a user-supplied path filter 1618 * @return an array of FileStatus objects 1619 * @throws IOException if any I/O error occurs when fetching file status 1620 */ 1621 public FileStatus[] globStatus(Path pathPattern, PathFilter filter) 1622 throws IOException { 1623 String filename = pathPattern.toUri().getPath(); 1624 List<FileStatus> allMatches = null; 1625 1626 List<String> filePatterns = GlobExpander.expand(filename); 1627 for (String filePattern : filePatterns) { 1628 Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern); 1629 List<FileStatus> matches = globStatusInternal(path, filter); 1630 if (matches != null) { 1631 if (allMatches == null) { 1632 allMatches = matches; 1633 } else { 1634 allMatches.addAll(matches); 1635 } 1636 } 1637 } 1638 1639 FileStatus[] results = null; 1640 if (allMatches != null) { 1641 results = allMatches.toArray(new FileStatus[allMatches.size()]); 1642 } else if (filePatterns.size() > 1) { 1643 // no matches with multiple expansions is a non-matching glob 1644 results = new FileStatus[0]; 1645 } 1646 return results; 1647 } 1648 1649 // sort gripes because FileStatus Comparable isn't parameterized... 1650 @SuppressWarnings("unchecked") 1651 private List<FileStatus> globStatusInternal(Path pathPattern, 1652 PathFilter filter) throws IOException { 1653 boolean patternHasGlob = false; // pathPattern has any globs 1654 List<FileStatus> matches = new ArrayList<FileStatus>(); 1655 1656 // determine starting point 1657 int level = 0; 1658 String baseDir = Path.CUR_DIR; 1659 if (pathPattern.isAbsolute()) { 1660 level = 1; // need to skip empty item at beginning of split list 1661 baseDir = Path.SEPARATOR; 1662 } 1663 1664 // parse components and determine if it's a glob 1665 String[] components = null; 1666 GlobFilter[] filters = null; 1667 String filename = pathPattern.toUri().getPath(); 1668 if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) { 1669 components = filename.split(Path.SEPARATOR); 1670 filters = new GlobFilter[components.length]; 1671 for (int i=level; i < components.length; i++) { 1672 filters[i] = new GlobFilter(components[i]); 1673 patternHasGlob |= filters[i].hasPattern(); 1674 } 1675 if (!patternHasGlob) { 1676 baseDir = unquotePathComponent(filename); 1677 components = null; // short through to filter check 1678 } 1679 } 1680 1681 // seed the parent directory path, return if it doesn't exist 1682 try { 1683 matches.add(getFileStatus(new Path(baseDir))); 1684 } catch (FileNotFoundException e) { 1685 return patternHasGlob ? matches : null; 1686 } 1687 1688 // skip if there are no components other than the basedir 1689 if (components != null) { 1690 // iterate through each path component 1691 for (int i=level; (i < components.length) && !matches.isEmpty(); i++) { 1692 List<FileStatus> children = new ArrayList<FileStatus>(); 1693 for (FileStatus match : matches) { 1694 // don't look for children in a file matched by a glob 1695 if (!match.isDirectory()) { 1696 continue; 1697 } 1698 try { 1699 if (filters[i].hasPattern()) { 1700 // get all children matching the filter 1701 FileStatus[] statuses = listStatus(match.getPath(), filters[i]); 1702 children.addAll(Arrays.asList(statuses)); 1703 } else { 1704 // the component does not have a pattern 1705 String component = unquotePathComponent(components[i]); 1706 Path child = new Path(match.getPath(), component); 1707 children.add(getFileStatus(child)); 1708 } 1709 } catch (FileNotFoundException e) { 1710 // don't care 1711 } 1712 } 1713 matches = children; 1714 } 1715 } 1716 // remove anything that didn't match the filter 1717 if (!matches.isEmpty()) { 1718 Iterator<FileStatus> iter = matches.iterator(); 1719 while (iter.hasNext()) { 1720 if (!filter.accept(iter.next().getPath())) { 1721 iter.remove(); 1722 } 1723 } 1724 } 1725 // no final paths, if there were any globs return empty list 1726 if (matches.isEmpty()) { 1727 return patternHasGlob ? matches : null; 1728 } 1729 Collections.sort(matches); 1730 return matches; 1731 } 1732 1733 /** 1734 * The glob filter builds a regexp per path component. If the component 1735 * does not contain a shell metachar, then it falls back to appending the 1736 * raw string to the list of built up paths. This raw path needs to have 1737 * the quoting removed. Ie. convert all occurances of "\X" to "X" 1738 * @param name of the path component 1739 * @return the unquoted path component 1740 */ 1741 private String unquotePathComponent(String name) { 1742 return name.replaceAll("\\\\(.)", "$1"); 1743 } 1744 1745 /** 1746 * List the statuses of the files/directories in the given path if the path is 1747 * a directory. 1748 * Return the file's status and block locations If the path is a file. 1749 * 1750 * If a returned status is a file, it contains the file's block locations. 1751 * 1752 * @param f is the path 1753 * 1754 * @return an iterator that traverses statuses of the files/directories 1755 * in the given path 1756 * 1757 * @throws FileNotFoundException If <code>f</code> does not exist 1758 * @throws IOException If an I/O error occurred 1759 */ 1760 public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f) 1761 throws FileNotFoundException, IOException { 1762 return listLocatedStatus(f, DEFAULT_FILTER); 1763 } 1764 1765 /** 1766 * Listing a directory 1767 * The returned results include its block location if it is a file 1768 * The results are filtered by the given path filter 1769 * @param f a path 1770 * @param filter a path filter 1771 * @return an iterator that traverses statuses of the files/directories 1772 * in the given path 1773 * @throws FileNotFoundException if <code>f</code> does not exist 1774 * @throws IOException if any I/O error occurred 1775 */ 1776 protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f, 1777 final PathFilter filter) 1778 throws FileNotFoundException, IOException { 1779 return new RemoteIterator<LocatedFileStatus>() { 1780 private final FileStatus[] stats = listStatus(f, filter); 1781 private int i = 0; 1782 1783 @Override 1784 public boolean hasNext() { 1785 return i<stats.length; 1786 } 1787 1788 @Override 1789 public LocatedFileStatus next() throws IOException { 1790 if (!hasNext()) { 1791 throw new NoSuchElementException("No more entry in " + f); 1792 } 1793 FileStatus result = stats[i++]; 1794 BlockLocation[] locs = result.isFile() ? 1795 getFileBlockLocations(result.getPath(), 0, result.getLen()) : 1796 null; 1797 return new LocatedFileStatus(result, locs); 1798 } 1799 }; 1800 } 1801 1802 /** 1803 * List the statuses and block locations of the files in the given path. 1804 * 1805 * If the path is a directory, 1806 * if recursive is false, returns files in the directory; 1807 * if recursive is true, return files in the subtree rooted at the path. 1808 * If the path is a file, return the file's status and block locations. 1809 * 1810 * @param f is the path 1811 * @param recursive if the subdirectories need to be traversed recursively 1812 * 1813 * @return an iterator that traverses statuses of the files 1814 * 1815 * @throws FileNotFoundException when the path does not exist; 1816 * IOException see specific implementation 1817 */ 1818 public RemoteIterator<LocatedFileStatus> listFiles( 1819 final Path f, final boolean recursive) 1820 throws FileNotFoundException, IOException { 1821 return new RemoteIterator<LocatedFileStatus>() { 1822 private Stack<RemoteIterator<LocatedFileStatus>> itors = 1823 new Stack<RemoteIterator<LocatedFileStatus>>(); 1824 private RemoteIterator<LocatedFileStatus> curItor = 1825 listLocatedStatus(f); 1826 private LocatedFileStatus curFile; 1827 1828 @Override 1829 public boolean hasNext() throws IOException { 1830 while (curFile == null) { 1831 if (curItor.hasNext()) { 1832 handleFileStat(curItor.next()); 1833 } else if (!itors.empty()) { 1834 curItor = itors.pop(); 1835 } else { 1836 return false; 1837 } 1838 } 1839 return true; 1840 } 1841 1842 /** 1843 * Process the input stat. 1844 * If it is a file, return the file stat. 1845 * If it is a directory, traverse the directory if recursive is true; 1846 * ignore it if recursive is false. 1847 * @param stat input status 1848 * @throws IOException if any IO error occurs 1849 */ 1850 private void handleFileStat(LocatedFileStatus stat) throws IOException { 1851 if (stat.isFile()) { // file 1852 curFile = stat; 1853 } else if (recursive) { // directory 1854 itors.push(curItor); 1855 curItor = listLocatedStatus(stat.getPath()); 1856 } 1857 } 1858 1859 @Override 1860 public LocatedFileStatus next() throws IOException { 1861 if (hasNext()) { 1862 LocatedFileStatus result = curFile; 1863 curFile = null; 1864 return result; 1865 } 1866 throw new java.util.NoSuchElementException("No more entry in " + f); 1867 } 1868 }; 1869 } 1870 1871 /** Return the current user's home directory in this filesystem. 1872 * The default implementation returns "/user/$USER/". 1873 */ 1874 public Path getHomeDirectory() { 1875 return this.makeQualified( 1876 new Path("/user/"+System.getProperty("user.name"))); 1877 } 1878 1879 1880 /** 1881 * Set the current working directory for the given file system. All relative 1882 * paths will be resolved relative to it. 1883 * 1884 * @param new_dir 1885 */ 1886 public abstract void setWorkingDirectory(Path new_dir); 1887 1888 /** 1889 * Get the current working directory for the given file system 1890 * @return the directory pathname 1891 */ 1892 public abstract Path getWorkingDirectory(); 1893 1894 1895 /** 1896 * Note: with the new FilesContext class, getWorkingDirectory() 1897 * will be removed. 1898 * The working directory is implemented in FilesContext. 1899 * 1900 * Some file systems like LocalFileSystem have an initial workingDir 1901 * that we use as the starting workingDir. For other file systems 1902 * like HDFS there is no built in notion of an inital workingDir. 1903 * 1904 * @return if there is built in notion of workingDir then it 1905 * is returned; else a null is returned. 1906 */ 1907 protected Path getInitialWorkingDirectory() { 1908 return null; 1909 } 1910 1911 /** 1912 * Call {@link #mkdirs(Path, FsPermission)} with default permission. 1913 */ 1914 public boolean mkdirs(Path f) throws IOException { 1915 return mkdirs(f, FsPermission.getDirDefault()); 1916 } 1917 1918 /** 1919 * Make the given file and all non-existent parents into 1920 * directories. Has the semantics of Unix 'mkdir -p'. 1921 * Existence of the directory hierarchy is not an error. 1922 * @param f path to create 1923 * @param permission to apply to f 1924 */ 1925 public abstract boolean mkdirs(Path f, FsPermission permission 1926 ) throws IOException; 1927 1928 /** 1929 * The src file is on the local disk. Add it to FS at 1930 * the given dst name and the source is kept intact afterwards 1931 * @param src path 1932 * @param dst path 1933 */ 1934 public void copyFromLocalFile(Path src, Path dst) 1935 throws IOException { 1936 copyFromLocalFile(false, src, dst); 1937 } 1938 1939 /** 1940 * The src files is on the local disk. Add it to FS at 1941 * the given dst name, removing the source afterwards. 1942 * @param srcs path 1943 * @param dst path 1944 */ 1945 public void moveFromLocalFile(Path[] srcs, Path dst) 1946 throws IOException { 1947 copyFromLocalFile(true, true, srcs, dst); 1948 } 1949 1950 /** 1951 * The src file is on the local disk. Add it to FS at 1952 * the given dst name, removing the source afterwards. 1953 * @param src path 1954 * @param dst path 1955 */ 1956 public void moveFromLocalFile(Path src, Path dst) 1957 throws IOException { 1958 copyFromLocalFile(true, src, dst); 1959 } 1960 1961 /** 1962 * The src file is on the local disk. Add it to FS at 1963 * the given dst name. 1964 * delSrc indicates if the source should be removed 1965 * @param delSrc whether to delete the src 1966 * @param src path 1967 * @param dst path 1968 */ 1969 public void copyFromLocalFile(boolean delSrc, Path src, Path dst) 1970 throws IOException { 1971 copyFromLocalFile(delSrc, true, src, dst); 1972 } 1973 1974 /** 1975 * The src files are on the local disk. Add it to FS at 1976 * the given dst name. 1977 * delSrc indicates if the source should be removed 1978 * @param delSrc whether to delete the src 1979 * @param overwrite whether to overwrite an existing file 1980 * @param srcs array of paths which are source 1981 * @param dst path 1982 */ 1983 public void copyFromLocalFile(boolean delSrc, boolean overwrite, 1984 Path[] srcs, Path dst) 1985 throws IOException { 1986 Configuration conf = getConf(); 1987 FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf); 1988 } 1989 1990 /** 1991 * The src file is on the local disk. Add it to FS at 1992 * the given dst name. 1993 * delSrc indicates if the source should be removed 1994 * @param delSrc whether to delete the src 1995 * @param overwrite whether to overwrite an existing file 1996 * @param src path 1997 * @param dst path 1998 */ 1999 public void copyFromLocalFile(boolean delSrc, boolean overwrite, 2000 Path src, Path dst) 2001 throws IOException { 2002 Configuration conf = getConf(); 2003 FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf); 2004 } 2005 2006 /** 2007 * The src file is under FS, and the dst is on the local disk. 2008 * Copy it from FS control to the local dst name. 2009 * @param src path 2010 * @param dst path 2011 */ 2012 public void copyToLocalFile(Path src, Path dst) throws IOException { 2013 copyToLocalFile(false, src, dst); 2014 } 2015 2016 /** 2017 * The src file is under FS, and the dst is on the local disk. 2018 * Copy it from FS control to the local dst name. 2019 * Remove the source afterwards 2020 * @param src path 2021 * @param dst path 2022 */ 2023 public void moveToLocalFile(Path src, Path dst) throws IOException { 2024 copyToLocalFile(true, src, dst); 2025 } 2026 2027 /** 2028 * The src file is under FS, and the dst is on the local disk. 2029 * Copy it from FS control to the local dst name. 2030 * delSrc indicates if the src will be removed or not. 2031 * @param delSrc whether to delete the src 2032 * @param src path 2033 * @param dst path 2034 */ 2035 public void copyToLocalFile(boolean delSrc, Path src, Path dst) 2036 throws IOException { 2037 copyToLocalFile(delSrc, src, dst, false); 2038 } 2039 2040 /** 2041 * The src file is under FS, and the dst is on the local disk. Copy it from FS 2042 * control to the local dst name. delSrc indicates if the src will be removed 2043 * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem 2044 * as local file system or not. RawLocalFileSystem is non crc file system.So, 2045 * It will not create any crc files at local. 2046 * 2047 * @param delSrc 2048 * whether to delete the src 2049 * @param src 2050 * path 2051 * @param dst 2052 * path 2053 * @param useRawLocalFileSystem 2054 * whether to use RawLocalFileSystem as local file system or not. 2055 * 2056 * @throws IOException 2057 * - if any IO error 2058 */ 2059 public void copyToLocalFile(boolean delSrc, Path src, Path dst, 2060 boolean useRawLocalFileSystem) throws IOException { 2061 Configuration conf = getConf(); 2062 FileSystem local = null; 2063 if (useRawLocalFileSystem) { 2064 local = getLocal(conf).getRawFileSystem(); 2065 } else { 2066 local = getLocal(conf); 2067 } 2068 FileUtil.copy(this, src, local, dst, delSrc, conf); 2069 } 2070 2071 /** 2072 * Returns a local File that the user can write output to. The caller 2073 * provides both the eventual FS target name and the local working 2074 * file. If the FS is local, we write directly into the target. If 2075 * the FS is remote, we write into the tmp local area. 2076 * @param fsOutputFile path of output file 2077 * @param tmpLocalFile path of local tmp file 2078 */ 2079 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) 2080 throws IOException { 2081 return tmpLocalFile; 2082 } 2083 2084 /** 2085 * Called when we're all done writing to the target. A local FS will 2086 * do nothing, because we've written to exactly the right place. A remote 2087 * FS will copy the contents of tmpLocalFile to the correct target at 2088 * fsOutputFile. 2089 * @param fsOutputFile path of output file 2090 * @param tmpLocalFile path to local tmp file 2091 */ 2092 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) 2093 throws IOException { 2094 moveFromLocalFile(tmpLocalFile, fsOutputFile); 2095 } 2096 2097 /** 2098 * No more filesystem operations are needed. Will 2099 * release any held locks. 2100 */ 2101 public void close() throws IOException { 2102 // delete all files that were marked as delete-on-exit. 2103 processDeleteOnExit(); 2104 CACHE.remove(this.key, this); 2105 } 2106 2107 /** Return the total size of all files in the filesystem.*/ 2108 public long getUsed() throws IOException{ 2109 long used = 0; 2110 FileStatus[] files = listStatus(new Path("/")); 2111 for(FileStatus file:files){ 2112 used += file.getLen(); 2113 } 2114 return used; 2115 } 2116 2117 /** 2118 * Get the block size for a particular file. 2119 * @param f the filename 2120 * @return the number of bytes in a block 2121 */ 2122 /** @deprecated Use getFileStatus() instead */ 2123 @Deprecated 2124 public long getBlockSize(Path f) throws IOException { 2125 return getFileStatus(f).getBlockSize(); 2126 } 2127 2128 /** 2129 * Return the number of bytes that large input files should be optimally 2130 * be split into to minimize i/o time. 2131 * @deprecated use {@link #getDefaultBlockSize(Path)} instead 2132 */ 2133 @Deprecated 2134 public long getDefaultBlockSize() { 2135 // default to 32MB: large enough to minimize the impact of seeks 2136 return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); 2137 } 2138 2139 /** Return the number of bytes that large input files should be optimally 2140 * be split into to minimize i/o time. The given path will be used to 2141 * locate the actual filesystem. The full path does not have to exist. 2142 * @param f path of file 2143 * @return the default block size for the path's filesystem 2144 */ 2145 public long getDefaultBlockSize(Path f) { 2146 return getDefaultBlockSize(); 2147 } 2148 2149 /** 2150 * Get the default replication. 2151 * @deprecated use {@link #getDefaultReplication(Path)} instead 2152 */ 2153 @Deprecated 2154 public short getDefaultReplication() { return 1; } 2155 2156 /** 2157 * Get the default replication for a path. The given path will be used to 2158 * locate the actual filesystem. The full path does not have to exist. 2159 * @param path of the file 2160 * @return default replication for the path's filesystem 2161 */ 2162 public short getDefaultReplication(Path path) { 2163 return getDefaultReplication(); 2164 } 2165 2166 /** 2167 * Return a file status object that represents the path. 2168 * @param f The path we want information from 2169 * @return a FileStatus object 2170 * @throws FileNotFoundException when the path does not exist; 2171 * IOException see specific implementation 2172 */ 2173 public abstract FileStatus getFileStatus(Path f) throws IOException; 2174 2175 /** 2176 * Get the checksum of a file. 2177 * 2178 * @param f The file path 2179 * @return The file checksum. The default return value is null, 2180 * which indicates that no checksum algorithm is implemented 2181 * in the corresponding FileSystem. 2182 */ 2183 public FileChecksum getFileChecksum(Path f) throws IOException { 2184 return null; 2185 } 2186 2187 /** 2188 * Set the verify checksum flag. This is only applicable if the 2189 * corresponding FileSystem supports checksum. By default doesn't do anything. 2190 * @param verifyChecksum 2191 */ 2192 public void setVerifyChecksum(boolean verifyChecksum) { 2193 //doesn't do anything 2194 } 2195 2196 /** 2197 * Set the write checksum flag. This is only applicable if the 2198 * corresponding FileSystem supports checksum. By default doesn't do anything. 2199 * @param writeChecksum 2200 */ 2201 public void setWriteChecksum(boolean writeChecksum) { 2202 //doesn't do anything 2203 } 2204 2205 /** 2206 * Returns a status object describing the use and capacity of the 2207 * file system. If the file system has multiple partitions, the 2208 * use and capacity of the root partition is reflected. 2209 * 2210 * @return a FsStatus object 2211 * @throws IOException 2212 * see specific implementation 2213 */ 2214 public FsStatus getStatus() throws IOException { 2215 return getStatus(null); 2216 } 2217 2218 /** 2219 * Returns a status object describing the use and capacity of the 2220 * file system. If the file system has multiple partitions, the 2221 * use and capacity of the partition pointed to by the specified 2222 * path is reflected. 2223 * @param p Path for which status should be obtained. null means 2224 * the default partition. 2225 * @return a FsStatus object 2226 * @throws IOException 2227 * see specific implementation 2228 */ 2229 public FsStatus getStatus(Path p) throws IOException { 2230 return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE); 2231 } 2232 2233 /** 2234 * Set permission of a path. 2235 * @param p 2236 * @param permission 2237 */ 2238 public void setPermission(Path p, FsPermission permission 2239 ) throws IOException { 2240 } 2241 2242 /** 2243 * Set owner of a path (i.e. a file or a directory). 2244 * The parameters username and groupname cannot both be null. 2245 * @param p The path 2246 * @param username If it is null, the original username remains unchanged. 2247 * @param groupname If it is null, the original groupname remains unchanged. 2248 */ 2249 public void setOwner(Path p, String username, String groupname 2250 ) throws IOException { 2251 } 2252 2253 /** 2254 * Set access time of a file 2255 * @param p The path 2256 * @param mtime Set the modification time of this file. 2257 * The number of milliseconds since Jan 1, 1970. 2258 * A value of -1 means that this call should not set modification time. 2259 * @param atime Set the access time of this file. 2260 * The number of milliseconds since Jan 1, 1970. 2261 * A value of -1 means that this call should not set access time. 2262 */ 2263 public void setTimes(Path p, long mtime, long atime 2264 ) throws IOException { 2265 } 2266 2267 // making it volatile to be able to do a double checked locking 2268 private volatile static boolean FILE_SYSTEMS_LOADED = false; 2269 2270 private static final Map<String, Class<? extends FileSystem>> 2271 SERVICE_FILE_SYSTEMS = new HashMap<String, Class<? extends FileSystem>>(); 2272 2273 private static void loadFileSystems() { 2274 synchronized (FileSystem.class) { 2275 if (!FILE_SYSTEMS_LOADED) { 2276 ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class); 2277 for (FileSystem fs : serviceLoader) { 2278 SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); 2279 } 2280 FILE_SYSTEMS_LOADED = true; 2281 } 2282 } 2283 } 2284 2285 public static Class<? extends FileSystem> getFileSystemClass(String scheme, 2286 Configuration conf) throws IOException { 2287 if (!FILE_SYSTEMS_LOADED) { 2288 loadFileSystems(); 2289 } 2290 Class<? extends FileSystem> clazz = null; 2291 if (conf != null) { 2292 clazz = (Class<? extends FileSystem>) conf.getClass("fs." + scheme + ".impl", null); 2293 } 2294 if (clazz == null) { 2295 clazz = SERVICE_FILE_SYSTEMS.get(scheme); 2296 } 2297 if (clazz == null) { 2298 throw new IOException("No FileSystem for scheme: " + scheme); 2299 } 2300 return clazz; 2301 } 2302 2303 private static FileSystem createFileSystem(URI uri, Configuration conf 2304 ) throws IOException { 2305 Class<?> clazz = getFileSystemClass(uri.getScheme(), conf); 2306 if (clazz == null) { 2307 throw new IOException("No FileSystem for scheme: " + uri.getScheme()); 2308 } 2309 FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf); 2310 fs.initialize(uri, conf); 2311 return fs; 2312 } 2313 2314 /** Caching FileSystem objects */ 2315 static class Cache { 2316 private final ClientFinalizer clientFinalizer = new ClientFinalizer(); 2317 2318 private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>(); 2319 private final Set<Key> toAutoClose = new HashSet<Key>(); 2320 2321 /** A variable that makes all objects in the cache unique */ 2322 private static AtomicLong unique = new AtomicLong(1); 2323 2324 FileSystem get(URI uri, Configuration conf) throws IOException{ 2325 Key key = new Key(uri, conf); 2326 return getInternal(uri, conf, key); 2327 } 2328 2329 /** The objects inserted into the cache using this method are all unique */ 2330 FileSystem getUnique(URI uri, Configuration conf) throws IOException{ 2331 Key key = new Key(uri, conf, unique.getAndIncrement()); 2332 return getInternal(uri, conf, key); 2333 } 2334 2335 private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException{ 2336 FileSystem fs; 2337 synchronized (this) { 2338 fs = map.get(key); 2339 } 2340 if (fs != null) { 2341 return fs; 2342 } 2343 2344 fs = createFileSystem(uri, conf); 2345 synchronized (this) { // refetch the lock again 2346 FileSystem oldfs = map.get(key); 2347 if (oldfs != null) { // a file system is created while lock is releasing 2348 fs.close(); // close the new file system 2349 return oldfs; // return the old file system 2350 } 2351 2352 // now insert the new file system into the map 2353 if (map.isEmpty() 2354 && !ShutdownHookManager.get().isShutdownInProgress()) { 2355 ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY); 2356 } 2357 fs.key = key; 2358 map.put(key, fs); 2359 if (conf.getBoolean("fs.automatic.close", true)) { 2360 toAutoClose.add(key); 2361 } 2362 return fs; 2363 } 2364 } 2365 2366 synchronized void remove(Key key, FileSystem fs) { 2367 if (map.containsKey(key) && fs == map.get(key)) { 2368 map.remove(key); 2369 toAutoClose.remove(key); 2370 } 2371 } 2372 2373 synchronized void closeAll() throws IOException { 2374 closeAll(false); 2375 } 2376 2377 /** 2378 * Close all FileSystem instances in the Cache. 2379 * @param onlyAutomatic only close those that are marked for automatic closing 2380 */ 2381 synchronized void closeAll(boolean onlyAutomatic) throws IOException { 2382 List<IOException> exceptions = new ArrayList<IOException>(); 2383 2384 // Make a copy of the keys in the map since we'll be modifying 2385 // the map while iterating over it, which isn't safe. 2386 List<Key> keys = new ArrayList<Key>(); 2387 keys.addAll(map.keySet()); 2388 2389 for (Key key : keys) { 2390 final FileSystem fs = map.get(key); 2391 2392 if (onlyAutomatic && !toAutoClose.contains(key)) { 2393 continue; 2394 } 2395 2396 //remove from cache 2397 remove(key, fs); 2398 2399 if (fs != null) { 2400 try { 2401 fs.close(); 2402 } 2403 catch(IOException ioe) { 2404 exceptions.add(ioe); 2405 } 2406 } 2407 } 2408 2409 if (!exceptions.isEmpty()) { 2410 throw MultipleIOException.createIOException(exceptions); 2411 } 2412 } 2413 2414 private class ClientFinalizer implements Runnable { 2415 public synchronized void run() { 2416 try { 2417 closeAll(true); 2418 } catch (IOException e) { 2419 LOG.info("FileSystem.Cache.closeAll() threw an exception:\n" + e); 2420 } 2421 } 2422 } 2423 2424 synchronized void closeAll(UserGroupInformation ugi) throws IOException { 2425 List<FileSystem> targetFSList = new ArrayList<FileSystem>(); 2426 //Make a pass over the list and collect the filesystems to close 2427 //we cannot close inline since close() removes the entry from the Map 2428 for (Map.Entry<Key, FileSystem> entry : map.entrySet()) { 2429 final Key key = entry.getKey(); 2430 final FileSystem fs = entry.getValue(); 2431 if (ugi.equals(key.ugi) && fs != null) { 2432 targetFSList.add(fs); 2433 } 2434 } 2435 List<IOException> exceptions = new ArrayList<IOException>(); 2436 //now make a pass over the target list and close each 2437 for (FileSystem fs : targetFSList) { 2438 try { 2439 fs.close(); 2440 } 2441 catch(IOException ioe) { 2442 exceptions.add(ioe); 2443 } 2444 } 2445 if (!exceptions.isEmpty()) { 2446 throw MultipleIOException.createIOException(exceptions); 2447 } 2448 } 2449 2450 /** FileSystem.Cache.Key */ 2451 static class Key { 2452 final String scheme; 2453 final String authority; 2454 final UserGroupInformation ugi; 2455 final long unique; // an artificial way to make a key unique 2456 2457 Key(URI uri, Configuration conf) throws IOException { 2458 this(uri, conf, 0); 2459 } 2460 2461 Key(URI uri, Configuration conf, long unique) throws IOException { 2462 scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase(); 2463 authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase(); 2464 this.unique = unique; 2465 2466 this.ugi = UserGroupInformation.getCurrentUser(); 2467 } 2468 2469 /** {@inheritDoc} */ 2470 public int hashCode() { 2471 return (scheme + authority).hashCode() + ugi.hashCode() + (int)unique; 2472 } 2473 2474 static boolean isEqual(Object a, Object b) { 2475 return a == b || (a != null && a.equals(b)); 2476 } 2477 2478 /** {@inheritDoc} */ 2479 public boolean equals(Object obj) { 2480 if (obj == this) { 2481 return true; 2482 } 2483 if (obj != null && obj instanceof Key) { 2484 Key that = (Key)obj; 2485 return isEqual(this.scheme, that.scheme) 2486 && isEqual(this.authority, that.authority) 2487 && isEqual(this.ugi, that.ugi) 2488 && (this.unique == that.unique); 2489 } 2490 return false; 2491 } 2492 2493 /** {@inheritDoc} */ 2494 public String toString() { 2495 return "("+ugi.toString() + ")@" + scheme + "://" + authority; 2496 } 2497 } 2498 } 2499 2500 /** 2501 * Tracks statistics about how many reads, writes, and so forth have been 2502 * done in a FileSystem. 2503 * 2504 * Since there is only one of these objects per FileSystem, there will 2505 * typically be many threads writing to this object. Almost every operation 2506 * on an open file will involve a write to this object. In contrast, reading 2507 * statistics is done infrequently by most programs, and not at all by others. 2508 * Hence, this is optimized for writes. 2509 * 2510 * Each thread writes to its own thread-local area of memory. This removes 2511 * contention and allows us to scale up to many, many threads. To read 2512 * statistics, the reader thread totals up the contents of all of the 2513 * thread-local data areas. 2514 */ 2515 public static final class Statistics { 2516 /** 2517 * Statistics data. 2518 * 2519 * There is only a single writer to thread-local StatisticsData objects. 2520 * Hence, volatile is adequate here-- we do not need AtomicLong or similar 2521 * to prevent lost updates. 2522 * The Java specification guarantees that updates to volatile longs will 2523 * be perceived as atomic with respect to other threads, which is all we 2524 * need. 2525 */ 2526 private static class StatisticsData { 2527 volatile long bytesRead; 2528 volatile long bytesWritten; 2529 volatile int readOps; 2530 volatile int largeReadOps; 2531 volatile int writeOps; 2532 /** 2533 * Stores a weak reference to the thread owning this StatisticsData. 2534 * This allows us to remove StatisticsData objects that pertain to 2535 * threads that no longer exist. 2536 */ 2537 final WeakReference<Thread> owner; 2538 2539 StatisticsData(WeakReference<Thread> owner) { 2540 this.owner = owner; 2541 } 2542 2543 /** 2544 * Add another StatisticsData object to this one. 2545 */ 2546 void add(StatisticsData other) { 2547 this.bytesRead += other.bytesRead; 2548 this.bytesWritten += other.bytesWritten; 2549 this.readOps += other.readOps; 2550 this.largeReadOps += other.largeReadOps; 2551 this.writeOps += other.writeOps; 2552 } 2553 2554 /** 2555 * Negate the values of all statistics. 2556 */ 2557 void negate() { 2558 this.bytesRead = -this.bytesRead; 2559 this.bytesWritten = -this.bytesWritten; 2560 this.readOps = -this.readOps; 2561 this.largeReadOps = -this.largeReadOps; 2562 this.writeOps = -this.writeOps; 2563 } 2564 2565 @Override 2566 public String toString() { 2567 return bytesRead + " bytes read, " + bytesWritten + " bytes written, " 2568 + readOps + " read ops, " + largeReadOps + " large read ops, " 2569 + writeOps + " write ops"; 2570 } 2571 } 2572 2573 private interface StatisticsAggregator<T> { 2574 void accept(StatisticsData data); 2575 T aggregate(); 2576 } 2577 2578 private final String scheme; 2579 2580 /** 2581 * rootData is data that doesn't belong to any thread, but will be added 2582 * to the totals. This is useful for making copies of Statistics objects, 2583 * and for storing data that pertains to threads that have been garbage 2584 * collected. Protected by the Statistics lock. 2585 */ 2586 private final StatisticsData rootData; 2587 2588 /** 2589 * Thread-local data. 2590 */ 2591 private final ThreadLocal<StatisticsData> threadData; 2592 2593 /** 2594 * List of all thread-local data areas. Protected by the Statistics lock. 2595 */ 2596 private LinkedList<StatisticsData> allData; 2597 2598 public Statistics(String scheme) { 2599 this.scheme = scheme; 2600 this.rootData = new StatisticsData(null); 2601 this.threadData = new ThreadLocal<StatisticsData>(); 2602 this.allData = null; 2603 } 2604 2605 /** 2606 * Copy constructor. 2607 * 2608 * @param other The input Statistics object which is cloned. 2609 */ 2610 public Statistics(Statistics other) { 2611 this.scheme = other.scheme; 2612 this.rootData = new StatisticsData(null); 2613 other.visitAll(new StatisticsAggregator<Void>() { 2614 @Override 2615 public void accept(StatisticsData data) { 2616 rootData.add(data); 2617 } 2618 2619 public Void aggregate() { 2620 return null; 2621 } 2622 }); 2623 this.threadData = new ThreadLocal<StatisticsData>(); 2624 } 2625 2626 /** 2627 * Get or create the thread-local data associated with the current thread. 2628 */ 2629 private StatisticsData getThreadData() { 2630 StatisticsData data = threadData.get(); 2631 if (data == null) { 2632 data = new StatisticsData( 2633 new WeakReference<Thread>(Thread.currentThread())); 2634 threadData.set(data); 2635 synchronized(this) { 2636 if (allData == null) { 2637 allData = new LinkedList<StatisticsData>(); 2638 } 2639 allData.add(data); 2640 } 2641 } 2642 return data; 2643 } 2644 2645 /** 2646 * Increment the bytes read in the statistics 2647 * @param newBytes the additional bytes read 2648 */ 2649 public void incrementBytesRead(long newBytes) { 2650 getThreadData().bytesRead += newBytes; 2651 } 2652 2653 /** 2654 * Increment the bytes written in the statistics 2655 * @param newBytes the additional bytes written 2656 */ 2657 public void incrementBytesWritten(long newBytes) { 2658 getThreadData().bytesWritten += newBytes; 2659 } 2660 2661 /** 2662 * Increment the number of read operations 2663 * @param count number of read operations 2664 */ 2665 public void incrementReadOps(int count) { 2666 getThreadData().readOps += count; 2667 } 2668 2669 /** 2670 * Increment the number of large read operations 2671 * @param count number of large read operations 2672 */ 2673 public void incrementLargeReadOps(int count) { 2674 getThreadData().largeReadOps += count; 2675 } 2676 2677 /** 2678 * Increment the number of write operations 2679 * @param count number of write operations 2680 */ 2681 public void incrementWriteOps(int count) { 2682 getThreadData().writeOps += count; 2683 } 2684 2685 /** 2686 * Apply the given aggregator to all StatisticsData objects associated with 2687 * this Statistics object. 2688 * 2689 * For each StatisticsData object, we will call accept on the visitor. 2690 * Finally, at the end, we will call aggregate to get the final total. 2691 * 2692 * @param The visitor to use. 2693 * @return The total. 2694 */ 2695 private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) { 2696 visitor.accept(rootData); 2697 if (allData != null) { 2698 for (Iterator<StatisticsData> iter = allData.iterator(); 2699 iter.hasNext(); ) { 2700 StatisticsData data = iter.next(); 2701 visitor.accept(data); 2702 if (data.owner.get() == null) { 2703 /* 2704 * If the thread that created this thread-local data no 2705 * longer exists, remove the StatisticsData from our list 2706 * and fold the values into rootData. 2707 */ 2708 rootData.add(data); 2709 iter.remove(); 2710 } 2711 } 2712 } 2713 return visitor.aggregate(); 2714 } 2715 2716 /** 2717 * Get the total number of bytes read 2718 * @return the number of bytes 2719 */ 2720 public long getBytesRead() { 2721 return visitAll(new StatisticsAggregator<Long>() { 2722 private long bytesRead = 0; 2723 2724 @Override 2725 public void accept(StatisticsData data) { 2726 bytesRead += data.bytesRead; 2727 } 2728 2729 public Long aggregate() { 2730 return bytesRead; 2731 } 2732 }); 2733 } 2734 2735 /** 2736 * Get the total number of bytes written 2737 * @return the number of bytes 2738 */ 2739 public long getBytesWritten() { 2740 return visitAll(new StatisticsAggregator<Long>() { 2741 private long bytesWritten = 0; 2742 2743 @Override 2744 public void accept(StatisticsData data) { 2745 bytesWritten += data.bytesWritten; 2746 } 2747 2748 public Long aggregate() { 2749 return bytesWritten; 2750 } 2751 }); 2752 } 2753 2754 /** 2755 * Get the number of file system read operations such as list files 2756 * @return number of read operations 2757 */ 2758 public int getReadOps() { 2759 return visitAll(new StatisticsAggregator<Integer>() { 2760 private int readOps = 0; 2761 2762 @Override 2763 public void accept(StatisticsData data) { 2764 readOps += data.readOps; 2765 readOps += data.largeReadOps; 2766 } 2767 2768 public Integer aggregate() { 2769 return readOps; 2770 } 2771 }); 2772 } 2773 2774 /** 2775 * Get the number of large file system read operations such as list files 2776 * under a large directory 2777 * @return number of large read operations 2778 */ 2779 public int getLargeReadOps() { 2780 return visitAll(new StatisticsAggregator<Integer>() { 2781 private int largeReadOps = 0; 2782 2783 @Override 2784 public void accept(StatisticsData data) { 2785 largeReadOps += data.largeReadOps; 2786 } 2787 2788 public Integer aggregate() { 2789 return largeReadOps; 2790 } 2791 }); 2792 } 2793 2794 /** 2795 * Get the number of file system write operations such as create, append 2796 * rename etc. 2797 * @return number of write operations 2798 */ 2799 public int getWriteOps() { 2800 return visitAll(new StatisticsAggregator<Integer>() { 2801 private int writeOps = 0; 2802 2803 @Override 2804 public void accept(StatisticsData data) { 2805 writeOps += data.writeOps; 2806 } 2807 2808 public Integer aggregate() { 2809 return writeOps; 2810 } 2811 }); 2812 } 2813 2814 2815 @Override 2816 public String toString() { 2817 return visitAll(new StatisticsAggregator<String>() { 2818 private StatisticsData total = new StatisticsData(null); 2819 2820 @Override 2821 public void accept(StatisticsData data) { 2822 total.add(data); 2823 } 2824 2825 public String aggregate() { 2826 return total.toString(); 2827 } 2828 }); 2829 } 2830 2831 /** 2832 * Resets all statistics to 0. 2833 * 2834 * In order to reset, we add up all the thread-local statistics data, and 2835 * set rootData to the negative of that. 2836 * 2837 * This may seem like a counterintuitive way to reset the statsitics. Why 2838 * can't we just zero out all the thread-local data? Well, thread-local 2839 * data can only be modified by the thread that owns it. If we tried to 2840 * modify the thread-local data from this thread, our modification might get 2841 * interleaved with a read-modify-write operation done by the thread that 2842 * owns the data. That would result in our update getting lost. 2843 * 2844 * The approach used here avoids this problem because it only ever reads 2845 * (not writes) the thread-local data. Both reads and writes to rootData 2846 * are done under the lock, so we're free to modify rootData from any thread 2847 * that holds the lock. 2848 */ 2849 public void reset() { 2850 visitAll(new StatisticsAggregator<Void>() { 2851 private StatisticsData total = new StatisticsData(null); 2852 2853 @Override 2854 public void accept(StatisticsData data) { 2855 total.add(data); 2856 } 2857 2858 public Void aggregate() { 2859 total.negate(); 2860 rootData.add(total); 2861 return null; 2862 } 2863 }); 2864 } 2865 2866 /** 2867 * Get the uri scheme associated with this statistics object. 2868 * @return the schema associated with this set of statistics 2869 */ 2870 public String getScheme() { 2871 return scheme; 2872 } 2873 } 2874 2875 /** 2876 * Get the Map of Statistics object indexed by URI Scheme. 2877 * @return a Map having a key as URI scheme and value as Statistics object 2878 * @deprecated use {@link #getAllStatistics} instead 2879 */ 2880 @Deprecated 2881 public static synchronized Map<String, Statistics> getStatistics() { 2882 Map<String, Statistics> result = new HashMap<String, Statistics>(); 2883 for(Statistics stat: statisticsTable.values()) { 2884 result.put(stat.getScheme(), stat); 2885 } 2886 return result; 2887 } 2888 2889 /** 2890 * Return the FileSystem classes that have Statistics 2891 */ 2892 public static synchronized List<Statistics> getAllStatistics() { 2893 return new ArrayList<Statistics>(statisticsTable.values()); 2894 } 2895 2896 /** 2897 * Get the statistics for a particular file system 2898 * @param cls the class to lookup 2899 * @return a statistics object 2900 */ 2901 public static synchronized 2902 Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { 2903 Statistics result = statisticsTable.get(cls); 2904 if (result == null) { 2905 result = new Statistics(scheme); 2906 statisticsTable.put(cls, result); 2907 } 2908 return result; 2909 } 2910 2911 /** 2912 * Reset all statistics for all file systems 2913 */ 2914 public static synchronized void clearStatistics() { 2915 for(Statistics stat: statisticsTable.values()) { 2916 stat.reset(); 2917 } 2918 } 2919 2920 /** 2921 * Print all statistics for all file systems 2922 */ 2923 public static synchronized 2924 void printStatistics() throws IOException { 2925 for (Map.Entry<Class<? extends FileSystem>, Statistics> pair: 2926 statisticsTable.entrySet()) { 2927 System.out.println(" FileSystem " + pair.getKey().getName() + 2928 ": " + pair.getValue()); 2929 } 2930 } 2931 }