001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.fs; 020 021 import java.io.BufferedOutputStream; 022 import java.io.DataOutput; 023 import java.io.File; 024 import java.io.FileInputStream; 025 import java.io.FileNotFoundException; 026 import java.io.FileOutputStream; 027 import java.io.IOException; 028 import java.io.OutputStream; 029 import java.io.FileDescriptor; 030 import java.net.URI; 031 import java.nio.ByteBuffer; 032 import java.util.Arrays; 033 import java.util.EnumSet; 034 import java.util.StringTokenizer; 035 036 import org.apache.hadoop.classification.InterfaceAudience; 037 import org.apache.hadoop.classification.InterfaceStability; 038 import org.apache.hadoop.conf.Configuration; 039 import org.apache.hadoop.fs.permission.FsPermission; 040 import org.apache.hadoop.io.nativeio.NativeIO; 041 import org.apache.hadoop.util.Progressable; 042 import org.apache.hadoop.util.Shell; 043 import org.apache.hadoop.util.StringUtils; 044 045 /**************************************************************** 046 * Implement the FileSystem API for the raw local filesystem. 047 * 048 *****************************************************************/ 049 @InterfaceAudience.Public 050 @InterfaceStability.Stable 051 public class RawLocalFileSystem extends FileSystem { 052 static final URI NAME = URI.create("file:///"); 053 private Path workingDir; 054 055 public RawLocalFileSystem() { 056 workingDir = getInitialWorkingDirectory(); 057 } 058 059 private Path makeAbsolute(Path f) { 060 if (f.isAbsolute()) { 061 return f; 062 } else { 063 return new Path(workingDir, f); 064 } 065 } 066 067 /** Convert a path to a File. */ 068 public File pathToFile(Path path) { 069 checkPath(path); 070 if (!path.isAbsolute()) { 071 path = new Path(getWorkingDirectory(), path); 072 } 073 return new File(path.toUri().getPath()); 074 } 075 076 public URI getUri() { return NAME; } 077 078 public void initialize(URI uri, Configuration conf) throws IOException { 079 super.initialize(uri, conf); 080 setConf(conf); 081 } 082 083 class TrackingFileInputStream extends FileInputStream { 084 public TrackingFileInputStream(File f) throws IOException { 085 super(f); 086 } 087 088 public int read() throws IOException { 089 int result = super.read(); 090 if (result != -1) { 091 statistics.incrementBytesRead(1); 092 } 093 return result; 094 } 095 096 public int read(byte[] data) throws IOException { 097 int result = super.read(data); 098 if (result != -1) { 099 statistics.incrementBytesRead(result); 100 } 101 return result; 102 } 103 104 public int read(byte[] data, int offset, int length) throws IOException { 105 int result = super.read(data, offset, length); 106 if (result != -1) { 107 statistics.incrementBytesRead(result); 108 } 109 return result; 110 } 111 } 112 113 /******************************************************* 114 * For open()'s FSInputStream. 115 *******************************************************/ 116 class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor { 117 private FileInputStream fis; 118 private long position; 119 120 public LocalFSFileInputStream(Path f) throws IOException { 121 this.fis = new TrackingFileInputStream(pathToFile(f)); 122 } 123 124 public void seek(long pos) throws IOException { 125 fis.getChannel().position(pos); 126 this.position = pos; 127 } 128 129 public long getPos() throws IOException { 130 return this.position; 131 } 132 133 public boolean seekToNewSource(long targetPos) throws IOException { 134 return false; 135 } 136 137 /* 138 * Just forward to the fis 139 */ 140 public int available() throws IOException { return fis.available(); } 141 public void close() throws IOException { fis.close(); } 142 @Override 143 public boolean markSupported() { return false; } 144 145 public int read() throws IOException { 146 try { 147 int value = fis.read(); 148 if (value >= 0) { 149 this.position++; 150 } 151 return value; 152 } catch (IOException e) { // unexpected exception 153 throw new FSError(e); // assume native fs error 154 } 155 } 156 157 public int read(byte[] b, int off, int len) throws IOException { 158 try { 159 int value = fis.read(b, off, len); 160 if (value > 0) { 161 this.position += value; 162 } 163 return value; 164 } catch (IOException e) { // unexpected exception 165 throw new FSError(e); // assume native fs error 166 } 167 } 168 169 public int read(long position, byte[] b, int off, int len) 170 throws IOException { 171 ByteBuffer bb = ByteBuffer.wrap(b, off, len); 172 try { 173 return fis.getChannel().read(bb, position); 174 } catch (IOException e) { 175 throw new FSError(e); 176 } 177 } 178 179 public long skip(long n) throws IOException { 180 long value = fis.skip(n); 181 if (value > 0) { 182 this.position += value; 183 } 184 return value; 185 } 186 187 @Override 188 public FileDescriptor getFileDescriptor() throws IOException { 189 return fis.getFD(); 190 } 191 } 192 193 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 194 if (!exists(f)) { 195 throw new FileNotFoundException(f.toString()); 196 } 197 return new FSDataInputStream(new BufferedFSInputStream( 198 new LocalFSFileInputStream(f), bufferSize)); 199 } 200 201 /********************************************************* 202 * For create()'s FSOutputStream. 203 *********************************************************/ 204 class LocalFSFileOutputStream extends OutputStream { 205 private FileOutputStream fos; 206 207 private LocalFSFileOutputStream(Path f, boolean append) throws IOException { 208 this.fos = new FileOutputStream(pathToFile(f), append); 209 } 210 211 /* 212 * Just forward to the fos 213 */ 214 public void close() throws IOException { fos.close(); } 215 public void flush() throws IOException { fos.flush(); } 216 public void write(byte[] b, int off, int len) throws IOException { 217 try { 218 fos.write(b, off, len); 219 } catch (IOException e) { // unexpected exception 220 throw new FSError(e); // assume native fs error 221 } 222 } 223 224 public void write(int b) throws IOException { 225 try { 226 fos.write(b); 227 } catch (IOException e) { // unexpected exception 228 throw new FSError(e); // assume native fs error 229 } 230 } 231 } 232 233 /** {@inheritDoc} */ 234 public FSDataOutputStream append(Path f, int bufferSize, 235 Progressable progress) throws IOException { 236 if (!exists(f)) { 237 throw new FileNotFoundException("File " + f + " not found"); 238 } 239 if (getFileStatus(f).isDirectory()) { 240 throw new IOException("Cannot append to a diretory (=" + f + " )"); 241 } 242 return new FSDataOutputStream(new BufferedOutputStream( 243 new LocalFSFileOutputStream(f, true), bufferSize), statistics); 244 } 245 246 /** {@inheritDoc} */ 247 @Override 248 public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, 249 short replication, long blockSize, Progressable progress) 250 throws IOException { 251 return create(f, overwrite, true, bufferSize, replication, blockSize, progress); 252 } 253 254 private FSDataOutputStream create(Path f, boolean overwrite, 255 boolean createParent, int bufferSize, short replication, long blockSize, 256 Progressable progress) throws IOException { 257 if (exists(f) && !overwrite) { 258 throw new IOException("File already exists: "+f); 259 } 260 Path parent = f.getParent(); 261 if (parent != null && !mkdirs(parent)) { 262 throw new IOException("Mkdirs failed to create " + parent.toString()); 263 } 264 return new FSDataOutputStream(new BufferedOutputStream( 265 new LocalFSFileOutputStream(f, false), bufferSize), statistics); 266 } 267 268 @Override 269 @Deprecated 270 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, 271 EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, 272 Progressable progress) throws IOException { 273 if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) { 274 throw new IOException("File already exists: "+f); 275 } 276 return new FSDataOutputStream(new BufferedOutputStream( 277 new LocalFSFileOutputStream(f, false), bufferSize), statistics); 278 } 279 280 /** {@inheritDoc} */ 281 @Override 282 public FSDataOutputStream create(Path f, FsPermission permission, 283 boolean overwrite, int bufferSize, short replication, long blockSize, 284 Progressable progress) throws IOException { 285 286 FSDataOutputStream out = create(f, 287 overwrite, bufferSize, replication, blockSize, progress); 288 setPermission(f, permission); 289 return out; 290 } 291 292 /** {@inheritDoc} */ 293 @Override 294 public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, 295 boolean overwrite, 296 int bufferSize, short replication, long blockSize, 297 Progressable progress) throws IOException { 298 FSDataOutputStream out = create(f, 299 overwrite, false, bufferSize, replication, blockSize, progress); 300 setPermission(f, permission); 301 return out; 302 } 303 304 public boolean rename(Path src, Path dst) throws IOException { 305 if (pathToFile(src).renameTo(pathToFile(dst))) { 306 return true; 307 } 308 return FileUtil.copy(this, src, this, dst, true, getConf()); 309 } 310 311 /** 312 * Delete the given path to a file or directory. 313 * @param p the path to delete 314 * @param recursive to delete sub-directories 315 * @return true if the file or directory and all its contents were deleted 316 * @throws IOException if p is non-empty and recursive is false 317 */ 318 public boolean delete(Path p, boolean recursive) throws IOException { 319 File f = pathToFile(p); 320 if (f.isFile()) { 321 return f.delete(); 322 } else if (!recursive && f.isDirectory() && 323 (FileUtil.listFiles(f).length != 0)) { 324 throw new IOException("Directory " + f.toString() + " is not empty"); 325 } 326 return FileUtil.fullyDelete(f); 327 } 328 329 public FileStatus[] listStatus(Path f) throws IOException { 330 File localf = pathToFile(f); 331 FileStatus[] results; 332 333 if (!localf.exists()) { 334 throw new FileNotFoundException("File " + f + " does not exist"); 335 } 336 if (localf.isFile()) { 337 return new FileStatus[] { 338 new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) }; 339 } 340 341 File[] names = localf.listFiles(); 342 if (names == null) { 343 return null; 344 } 345 results = new FileStatus[names.length]; 346 int j = 0; 347 for (int i = 0; i < names.length; i++) { 348 try { 349 results[j] = getFileStatus(new Path(names[i].getAbsolutePath())); 350 j++; 351 } catch (FileNotFoundException e) { 352 // ignore the files not found since the dir list may have have changed 353 // since the names[] list was generated. 354 } 355 } 356 if (j == names.length) { 357 return results; 358 } 359 return Arrays.copyOf(results, j); 360 } 361 362 /** 363 * Creates the specified directory hierarchy. Does not 364 * treat existence as an error. 365 */ 366 public boolean mkdirs(Path f) throws IOException { 367 if(f == null) { 368 throw new IllegalArgumentException("mkdirs path arg is null"); 369 } 370 Path parent = f.getParent(); 371 File p2f = pathToFile(f); 372 if(parent != null) { 373 File parent2f = pathToFile(parent); 374 if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) { 375 throw new FileAlreadyExistsException("Parent path is not a directory: " 376 + parent); 377 } 378 } 379 return (parent == null || mkdirs(parent)) && 380 (p2f.mkdir() || p2f.isDirectory()); 381 } 382 383 /** {@inheritDoc} */ 384 @Override 385 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 386 boolean b = mkdirs(f); 387 if(b) { 388 setPermission(f, permission); 389 } 390 return b; 391 } 392 393 394 @Override 395 protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) 396 throws IOException { 397 boolean b = mkdirs(f); 398 setPermission(f, absolutePermission); 399 return b; 400 } 401 402 403 @Override 404 public Path getHomeDirectory() { 405 return this.makeQualified(new Path(System.getProperty("user.home"))); 406 } 407 408 /** 409 * Set the working directory to the given directory. 410 */ 411 @Override 412 public void setWorkingDirectory(Path newDir) { 413 workingDir = makeAbsolute(newDir); 414 checkPath(workingDir); 415 416 } 417 418 @Override 419 public Path getWorkingDirectory() { 420 return workingDir; 421 } 422 423 @Override 424 protected Path getInitialWorkingDirectory() { 425 return this.makeQualified(new Path(System.getProperty("user.dir"))); 426 } 427 428 /** {@inheritDoc} */ 429 @Override 430 public FsStatus getStatus(Path p) throws IOException { 431 File partition = pathToFile(p == null ? new Path("/") : p); 432 //File provides getUsableSpace() and getFreeSpace() 433 //File provides no API to obtain used space, assume used = total - free 434 return new FsStatus(partition.getTotalSpace(), 435 partition.getTotalSpace() - partition.getFreeSpace(), 436 partition.getFreeSpace()); 437 } 438 439 // In the case of the local filesystem, we can just rename the file. 440 public void moveFromLocalFile(Path src, Path dst) throws IOException { 441 rename(src, dst); 442 } 443 444 // We can write output directly to the final location 445 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) 446 throws IOException { 447 return fsOutputFile; 448 } 449 450 // It's in the right place - nothing to do. 451 public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile) 452 throws IOException { 453 } 454 455 public void close() throws IOException { 456 super.close(); 457 } 458 459 public String toString() { 460 return "LocalFS"; 461 } 462 463 public FileStatus getFileStatus(Path f) throws IOException { 464 File path = pathToFile(f); 465 if (path.exists()) { 466 return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this); 467 } else { 468 throw new FileNotFoundException("File " + f + " does not exist"); 469 } 470 } 471 472 static class RawLocalFileStatus extends FileStatus { 473 /* We can add extra fields here. It breaks at least CopyFiles.FilePair(). 474 * We recognize if the information is already loaded by check if 475 * onwer.equals(""). 476 */ 477 private boolean isPermissionLoaded() { 478 return !super.getOwner().equals(""); 479 } 480 481 RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { 482 super(f.length(), f.isDirectory(), 1, defaultBlockSize, 483 f.lastModified(), fs.makeQualified(new Path(f.getPath()))); 484 } 485 486 @Override 487 public FsPermission getPermission() { 488 if (!isPermissionLoaded()) { 489 loadPermissionInfo(); 490 } 491 return super.getPermission(); 492 } 493 494 @Override 495 public String getOwner() { 496 if (!isPermissionLoaded()) { 497 loadPermissionInfo(); 498 } 499 return super.getOwner(); 500 } 501 502 @Override 503 public String getGroup() { 504 if (!isPermissionLoaded()) { 505 loadPermissionInfo(); 506 } 507 return super.getGroup(); 508 } 509 510 /// loads permissions, owner, and group from `ls -ld` 511 private void loadPermissionInfo() { 512 IOException e = null; 513 try { 514 StringTokenizer t = new StringTokenizer( 515 execCommand(new File(getPath().toUri()), 516 Shell.getGET_PERMISSION_COMMAND())); 517 //expected format 518 //-rw------- 1 username groupname ... 519 String permission = t.nextToken(); 520 if (permission.length() > 10) { //files with ACLs might have a '+' 521 permission = permission.substring(0, 10); 522 } 523 setPermission(FsPermission.valueOf(permission)); 524 t.nextToken(); 525 setOwner(t.nextToken()); 526 setGroup(t.nextToken()); 527 } catch (Shell.ExitCodeException ioe) { 528 if (ioe.getExitCode() != 1) { 529 e = ioe; 530 } else { 531 setPermission(null); 532 setOwner(null); 533 setGroup(null); 534 } 535 } catch (IOException ioe) { 536 e = ioe; 537 } finally { 538 if (e != null) { 539 throw new RuntimeException("Error while running command to get " + 540 "file permissions : " + 541 StringUtils.stringifyException(e)); 542 } 543 } 544 } 545 546 @Override 547 public void write(DataOutput out) throws IOException { 548 if (!isPermissionLoaded()) { 549 loadPermissionInfo(); 550 } 551 super.write(out); 552 } 553 } 554 555 /** 556 * Use the command chown to set owner. 557 */ 558 @Override 559 public void setOwner(Path p, String username, String groupname) 560 throws IOException { 561 if (username == null && groupname == null) { 562 throw new IOException("username == null && groupname == null"); 563 } 564 565 if (username == null) { 566 execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 567 } else { 568 //OWNER[:[GROUP]] 569 String s = username + (groupname == null? "": ":" + groupname); 570 execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s); 571 } 572 } 573 574 /** 575 * Use the command chmod to set permission. 576 */ 577 @Override 578 public void setPermission(Path p, FsPermission permission) 579 throws IOException { 580 if (NativeIO.isAvailable()) { 581 NativeIO.chmod(pathToFile(p).getCanonicalPath(), 582 permission.toShort()); 583 } else { 584 execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND, 585 String.format("%05o", permission.toShort())); 586 } 587 } 588 589 private static String execCommand(File f, String... cmd) throws IOException { 590 String[] args = new String[cmd.length + 1]; 591 System.arraycopy(cmd, 0, args, 0, cmd.length); 592 args[cmd.length] = FileUtil.makeShellPath(f, true); 593 String output = Shell.execCommand(args); 594 return output; 595 } 596 597 }