001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.http.client; 019 020 import java.util.ArrayList; 021 import java.util.List; 022 import org.apache.hadoop.classification.InterfaceAudience; 023 import org.apache.hadoop.conf.Configuration; 024 import org.apache.hadoop.fs.ContentSummary; 025 import org.apache.hadoop.fs.DelegationTokenRenewer; 026 import org.apache.hadoop.fs.FSDataInputStream; 027 import org.apache.hadoop.fs.FSDataOutputStream; 028 import org.apache.hadoop.fs.FileChecksum; 029 import org.apache.hadoop.fs.FileStatus; 030 import org.apache.hadoop.fs.FileSystem; 031 import org.apache.hadoop.fs.Path; 032 import org.apache.hadoop.fs.PositionedReadable; 033 import org.apache.hadoop.fs.Seekable; 034 import org.apache.hadoop.fs.permission.FsPermission; 035 import org.apache.hadoop.hdfs.DFSConfigKeys; 036 import org.apache.hadoop.net.NetUtils; 037 import org.apache.hadoop.security.UserGroupInformation; 038 import org.apache.hadoop.security.authentication.client.AuthenticatedURL; 039 import org.apache.hadoop.security.authentication.client.Authenticator; 040 import org.apache.hadoop.security.token.Token; 041 import org.apache.hadoop.security.token.TokenIdentifier; 042 import org.apache.hadoop.util.Progressable; 043 import org.apache.hadoop.util.ReflectionUtils; 044 import org.apache.hadoop.util.StringUtils; 045 import org.json.simple.JSONArray; 046 import org.json.simple.JSONObject; 047 048 import java.io.BufferedInputStream; 049 import java.io.BufferedOutputStream; 050 import java.io.DataInput; 051 import java.io.DataOutput; 052 import java.io.FileNotFoundException; 053 import java.io.FilterInputStream; 054 import java.io.IOException; 055 import java.io.InputStream; 056 import java.io.OutputStream; 057 import java.net.HttpURLConnection; 058 import java.net.InetSocketAddress; 059 import java.net.URI; 060 import java.net.URISyntaxException; 061 import java.net.URL; 062 import java.security.PrivilegedExceptionAction; 063 import java.text.MessageFormat; 064 import java.util.HashMap; 065 import java.util.List; 066 import java.util.Map; 067 import java.util.concurrent.Callable; 068 069 /** 070 * HttpFSServer implementation of the FileSystemAccess FileSystem. 071 * <p/> 072 * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server. 073 */ 074 @InterfaceAudience.Private 075 public class HttpFSFileSystem extends FileSystem 076 implements DelegationTokenRenewer.Renewable { 077 078 public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME; 079 080 public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION; 081 082 public static final String SCHEME = "webhdfs"; 083 084 public static final String OP_PARAM = "op"; 085 public static final String DO_AS_PARAM = "doas"; 086 public static final String OVERWRITE_PARAM = "overwrite"; 087 public static final String REPLICATION_PARAM = "replication"; 088 public static final String BLOCKSIZE_PARAM = "blocksize"; 089 public static final String PERMISSION_PARAM = "permission"; 090 public static final String DESTINATION_PARAM = "destination"; 091 public static final String RECURSIVE_PARAM = "recursive"; 092 public static final String SOURCES_PARAM = "sources"; 093 public static final String OWNER_PARAM = "owner"; 094 public static final String GROUP_PARAM = "group"; 095 public static final String MODIFICATION_TIME_PARAM = "modificationtime"; 096 public static final String ACCESS_TIME_PARAM = "accesstime"; 097 098 public static final Short DEFAULT_PERMISSION = 0755; 099 100 public static final String RENAME_JSON = "boolean"; 101 102 public static final String DELETE_JSON = "boolean"; 103 104 public static final String MKDIRS_JSON = "boolean"; 105 106 public static final String HOME_DIR_JSON = "Path"; 107 108 public static final String SET_REPLICATION_JSON = "boolean"; 109 110 public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream"; 111 112 public static enum FILE_TYPE { 113 FILE, DIRECTORY, SYMLINK; 114 115 public static FILE_TYPE getType(FileStatus fileStatus) { 116 if (fileStatus.isFile()) { 117 return FILE; 118 } 119 if (fileStatus.isDirectory()) { 120 return DIRECTORY; 121 } 122 if (fileStatus.isSymlink()) { 123 return SYMLINK; 124 } 125 throw new IllegalArgumentException("Could not determine filetype for: " + 126 fileStatus.getPath()); 127 } 128 } 129 130 public static final String FILE_STATUSES_JSON = "FileStatuses"; 131 public static final String FILE_STATUS_JSON = "FileStatus"; 132 public static final String PATH_SUFFIX_JSON = "pathSuffix"; 133 public static final String TYPE_JSON = "type"; 134 public static final String LENGTH_JSON = "length"; 135 public static final String OWNER_JSON = "owner"; 136 public static final String GROUP_JSON = "group"; 137 public static final String PERMISSION_JSON = "permission"; 138 public static final String ACCESS_TIME_JSON = "accessTime"; 139 public static final String MODIFICATION_TIME_JSON = "modificationTime"; 140 public static final String BLOCK_SIZE_JSON = "blockSize"; 141 public static final String REPLICATION_JSON = "replication"; 142 143 public static final String FILE_CHECKSUM_JSON = "FileChecksum"; 144 public static final String CHECKSUM_ALGORITHM_JSON = "algorithm"; 145 public static final String CHECKSUM_BYTES_JSON = "bytes"; 146 public static final String CHECKSUM_LENGTH_JSON = "length"; 147 148 public static final String CONTENT_SUMMARY_JSON = "ContentSummary"; 149 public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount"; 150 public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount"; 151 public static final String CONTENT_SUMMARY_LENGTH_JSON = "length"; 152 public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota"; 153 public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed"; 154 public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota"; 155 156 public static final String ERROR_JSON = "RemoteException"; 157 public static final String ERROR_EXCEPTION_JSON = "exception"; 158 public static final String ERROR_CLASSNAME_JSON = "javaClassName"; 159 public static final String ERROR_MESSAGE_JSON = "message"; 160 161 public static final int HTTP_TEMPORARY_REDIRECT = 307; 162 163 private static final String HTTP_GET = "GET"; 164 private static final String HTTP_PUT = "PUT"; 165 private static final String HTTP_POST = "POST"; 166 private static final String HTTP_DELETE = "DELETE"; 167 168 @InterfaceAudience.Private 169 public static enum Operation { 170 OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET), 171 GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), 172 GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), 173 INSTRUMENTATION(HTTP_GET), 174 APPEND(HTTP_POST), CONCAT(HTTP_POST), 175 CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), 176 SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), 177 DELETE(HTTP_DELETE); 178 179 private String httpMethod; 180 181 Operation(String httpMethod) { 182 this.httpMethod = httpMethod; 183 } 184 185 public String getMethod() { 186 return httpMethod; 187 } 188 189 } 190 191 192 private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); 193 private URI uri; 194 private InetSocketAddress httpFSAddr; 195 private Path workingDir; 196 private UserGroupInformation realUser; 197 private String doAs; 198 private Token<?> delegationToken; 199 200 //This method enables handling UGI doAs with SPNEGO, we have to 201 //fallback to the realuser who logged in with Kerberos credentials 202 private <T> T doAsRealUserIfNecessary(final Callable<T> callable) 203 throws IOException { 204 try { 205 if (realUser.getShortUserName().equals(doAs)) { 206 return callable.call(); 207 } else { 208 return realUser.doAs(new PrivilegedExceptionAction<T>() { 209 @Override 210 public T run() throws Exception { 211 return callable.call(); 212 } 213 }); 214 } 215 } catch (Exception ex) { 216 throw new IOException(ex.toString(), ex); 217 } 218 } 219 220 /** 221 * Convenience method that creates a <code>HttpURLConnection</code> for the 222 * HttpFSServer file system operations. 223 * <p/> 224 * This methods performs and injects any needed authentication credentials 225 * via the {@link #getConnection(URL, String)} method 226 * 227 * @param method the HTTP method. 228 * @param params the query string parameters. 229 * @param path the file path 230 * @param makeQualified if the path should be 'makeQualified' 231 * 232 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, 233 * authenticated and ready to use for the specified path and file system operation. 234 * 235 * @throws IOException thrown if an IO error occurrs. 236 */ 237 private HttpURLConnection getConnection(final String method, 238 Map<String, String> params, Path path, boolean makeQualified) 239 throws IOException { 240 if (!realUser.getShortUserName().equals(doAs)) { 241 params.put(DO_AS_PARAM, doAs); 242 } 243 HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken); 244 if (makeQualified) { 245 path = makeQualified(path); 246 } 247 final URL url = HttpFSUtils.createHttpURL(path, params); 248 return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() { 249 @Override 250 public HttpURLConnection call() throws Exception { 251 return getConnection(url, method); 252 } 253 }); 254 } 255 256 /** 257 * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL. 258 * <p/> 259 * This methods performs and injects any needed authentication credentials. 260 * 261 * @param url url to connect to. 262 * @param method the HTTP method. 263 * 264 * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for 265 * the specified path and file system operation. 266 * 267 * @throws IOException thrown if an IO error occurrs. 268 */ 269 private HttpURLConnection getConnection(URL url, String method) throws IOException { 270 Class<? extends Authenticator> klass = 271 getConf().getClass("httpfs.authenticator.class", 272 HttpFSKerberosAuthenticator.class, Authenticator.class); 273 Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); 274 try { 275 HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); 276 conn.setRequestMethod(method); 277 if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) { 278 conn.setDoOutput(true); 279 } 280 return conn; 281 } catch (Exception ex) { 282 throw new IOException(ex); 283 } 284 } 285 286 /** 287 * Called after a new FileSystem instance is constructed. 288 * 289 * @param name a uri whose authority section names the host, port, etc. for this FileSystem 290 * @param conf the configuration 291 */ 292 @Override 293 public void initialize(URI name, Configuration conf) throws IOException { 294 UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 295 296 //the real use is the one that has the Kerberos credentials needed for 297 //SPNEGO to work 298 realUser = ugi.getRealUser(); 299 if (realUser == null) { 300 realUser = UserGroupInformation.getLoginUser(); 301 } 302 doAs = ugi.getShortUserName(); 303 super.initialize(name, conf); 304 try { 305 uri = new URI(name.getScheme() + "://" + name.getAuthority()); 306 httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString()); 307 } catch (URISyntaxException ex) { 308 throw new IOException(ex); 309 } 310 } 311 312 @Override 313 public String getScheme() { 314 return SCHEME; 315 } 316 317 /** 318 * Returns a URI whose scheme and authority identify this FileSystem. 319 * 320 * @return the URI whose scheme and authority identify this FileSystem. 321 */ 322 @Override 323 public URI getUri() { 324 return uri; 325 } 326 327 /** 328 * Get the default port for this file system. 329 * @return the default port or 0 if there isn't one 330 */ 331 @Override 332 protected int getDefaultPort() { 333 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, 334 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT); 335 } 336 337 /** 338 * HttpFSServer subclass of the <code>FSDataInputStream</code>. 339 * <p/> 340 * This implementation does not support the 341 * <code>PositionReadable</code> and <code>Seekable</code> methods. 342 */ 343 private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { 344 345 protected HttpFSDataInputStream(InputStream in, int bufferSize) { 346 super(new BufferedInputStream(in, bufferSize)); 347 } 348 349 @Override 350 public int read(long position, byte[] buffer, int offset, int length) throws IOException { 351 throw new UnsupportedOperationException(); 352 } 353 354 @Override 355 public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { 356 throw new UnsupportedOperationException(); 357 } 358 359 @Override 360 public void readFully(long position, byte[] buffer) throws IOException { 361 throw new UnsupportedOperationException(); 362 } 363 364 @Override 365 public void seek(long pos) throws IOException { 366 throw new UnsupportedOperationException(); 367 } 368 369 @Override 370 public long getPos() throws IOException { 371 throw new UnsupportedOperationException(); 372 } 373 374 @Override 375 public boolean seekToNewSource(long targetPos) throws IOException { 376 throw new UnsupportedOperationException(); 377 } 378 } 379 380 /** 381 * Opens an FSDataInputStream at the indicated Path. 382 * </p> 383 * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the 384 * <code>PositionReadable</code> and <code>Seekable</code> methods. 385 * 386 * @param f the file name to open 387 * @param bufferSize the size of the buffer to be used. 388 */ 389 @Override 390 public FSDataInputStream open(Path f, int bufferSize) throws IOException { 391 Map<String, String> params = new HashMap<String, String>(); 392 params.put(OP_PARAM, Operation.OPEN.toString()); 393 HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params, 394 f, true); 395 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 396 return new FSDataInputStream( 397 new HttpFSDataInputStream(conn.getInputStream(), bufferSize)); 398 } 399 400 /** 401 * HttpFSServer subclass of the <code>FSDataOutputStream</code>. 402 * <p/> 403 * This implementation closes the underlying HTTP connection validating the Http connection status 404 * at closing time. 405 */ 406 private static class HttpFSDataOutputStream extends FSDataOutputStream { 407 private HttpURLConnection conn; 408 private int closeStatus; 409 410 public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats) 411 throws IOException { 412 super(out, stats); 413 this.conn = conn; 414 this.closeStatus = closeStatus; 415 } 416 417 @Override 418 public void close() throws IOException { 419 try { 420 super.close(); 421 } finally { 422 HttpFSUtils.validateResponse(conn, closeStatus); 423 } 424 } 425 426 } 427 428 /** 429 * Converts a <code>FsPermission</code> to a Unix octal representation. 430 * 431 * @param p the permission. 432 * 433 * @return the Unix string symbolic reprentation. 434 */ 435 public static String permissionToString(FsPermission p) { 436 return Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8); 437 } 438 439 /* 440 * Common handling for uploading data for create and append operations. 441 */ 442 private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params, 443 int bufferSize, int expectedStatus) throws IOException { 444 HttpURLConnection conn = getConnection(method, params, f, true); 445 conn.setInstanceFollowRedirects(false); 446 boolean exceptionAlreadyHandled = false; 447 try { 448 if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) { 449 exceptionAlreadyHandled = true; 450 String location = conn.getHeaderField("Location"); 451 if (location != null) { 452 conn = getConnection(new URL(location), method); 453 conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE); 454 try { 455 OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); 456 return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics); 457 } catch (IOException ex) { 458 HttpFSUtils.validateResponse(conn, expectedStatus); 459 throw ex; 460 } 461 } else { 462 HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 463 throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]"); 464 } 465 } else { 466 throw new IOException( 467 MessageFormat.format("Expected HTTP status was [307], received [{0}]", 468 conn.getResponseCode())); 469 } 470 } catch (IOException ex) { 471 if (exceptionAlreadyHandled) { 472 throw ex; 473 } else { 474 HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT); 475 throw ex; 476 } 477 } 478 } 479 480 481 /** 482 * Opens an FSDataOutputStream at the indicated Path with write-progress 483 * reporting. 484 * <p/> 485 * IMPORTANT: The <code>Progressable</code> parameter is not used. 486 * 487 * @param f the file name to open. 488 * @param permission file permission. 489 * @param overwrite if a file with this name already exists, then if true, 490 * the file will be overwritten, and if false an error will be thrown. 491 * @param bufferSize the size of the buffer to be used. 492 * @param replication required block replication for the file. 493 * @param blockSize block size. 494 * @param progress progressable. 495 * 496 * @throws IOException 497 * @see #setPermission(Path, FsPermission) 498 */ 499 @Override 500 public FSDataOutputStream create(Path f, FsPermission permission, 501 boolean overwrite, int bufferSize, 502 short replication, long blockSize, 503 Progressable progress) throws IOException { 504 Map<String, String> params = new HashMap<String, String>(); 505 params.put(OP_PARAM, Operation.CREATE.toString()); 506 params.put(OVERWRITE_PARAM, Boolean.toString(overwrite)); 507 params.put(REPLICATION_PARAM, Short.toString(replication)); 508 params.put(BLOCKSIZE_PARAM, Long.toString(blockSize)); 509 params.put(PERMISSION_PARAM, permissionToString(permission)); 510 return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize, 511 HttpURLConnection.HTTP_CREATED); 512 } 513 514 515 /** 516 * Append to an existing file (optional operation). 517 * <p/> 518 * IMPORTANT: The <code>Progressable</code> parameter is not used. 519 * 520 * @param f the existing file to be appended. 521 * @param bufferSize the size of the buffer to be used. 522 * @param progress for reporting progress if it is not null. 523 * 524 * @throws IOException 525 */ 526 @Override 527 public FSDataOutputStream append(Path f, int bufferSize, 528 Progressable progress) throws IOException { 529 Map<String, String> params = new HashMap<String, String>(); 530 params.put(OP_PARAM, Operation.APPEND.toString()); 531 return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize, 532 HttpURLConnection.HTTP_OK); 533 } 534 535 /** 536 * Concat existing files together. 537 * @param f the path to the target destination. 538 * @param psrcs the paths to the sources to use for the concatenation. 539 * 540 * @throws IOException 541 */ 542 @Override 543 public void concat(Path f, Path[] psrcs) throws IOException { 544 List<String> strPaths = new ArrayList<String>(psrcs.length); 545 for(Path psrc : psrcs) { 546 strPaths.add(psrc.toUri().getPath()); 547 } 548 String srcs = StringUtils.join(",", strPaths); 549 550 Map<String, String> params = new HashMap<String, String>(); 551 params.put(OP_PARAM, Operation.CONCAT.toString()); 552 params.put(SOURCES_PARAM, srcs); 553 HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(), 554 params, f, true); 555 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 556 } 557 558 /** 559 * Renames Path src to Path dst. Can take place on local fs 560 * or remote DFS. 561 */ 562 @Override 563 public boolean rename(Path src, Path dst) throws IOException { 564 Map<String, String> params = new HashMap<String, String>(); 565 params.put(OP_PARAM, Operation.RENAME.toString()); 566 params.put(DESTINATION_PARAM, dst.toString()); 567 HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(), 568 params, src, true); 569 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 570 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 571 return (Boolean) json.get(RENAME_JSON); 572 } 573 574 /** 575 * Delete a file. 576 * 577 * @deprecated Use delete(Path, boolean) instead 578 */ 579 @SuppressWarnings({"deprecation"}) 580 @Deprecated 581 @Override 582 public boolean delete(Path f) throws IOException { 583 return delete(f, false); 584 } 585 586 /** 587 * Delete a file. 588 * 589 * @param f the path to delete. 590 * @param recursive if path is a directory and set to 591 * true, the directory is deleted else throws an exception. In 592 * case of a file the recursive can be set to either true or false. 593 * 594 * @return true if delete is successful else false. 595 * 596 * @throws IOException 597 */ 598 @Override 599 public boolean delete(Path f, boolean recursive) throws IOException { 600 Map<String, String> params = new HashMap<String, String>(); 601 params.put(OP_PARAM, Operation.DELETE.toString()); 602 params.put(RECURSIVE_PARAM, Boolean.toString(recursive)); 603 HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(), 604 params, f, true); 605 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 606 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 607 return (Boolean) json.get(DELETE_JSON); 608 } 609 610 /** 611 * List the statuses of the files/directories in the given path if the path is 612 * a directory. 613 * 614 * @param f given path 615 * 616 * @return the statuses of the files/directories in the given patch 617 * 618 * @throws IOException 619 */ 620 @Override 621 public FileStatus[] listStatus(Path f) throws IOException { 622 Map<String, String> params = new HashMap<String, String>(); 623 params.put(OP_PARAM, Operation.LISTSTATUS.toString()); 624 HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(), 625 params, f, true); 626 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 627 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 628 json = (JSONObject) json.get(FILE_STATUSES_JSON); 629 JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON); 630 FileStatus[] array = new FileStatus[jsonArray.size()]; 631 f = makeQualified(f); 632 for (int i = 0; i < jsonArray.size(); i++) { 633 array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i)); 634 } 635 return array; 636 } 637 638 /** 639 * Set the current working directory for the given file system. All relative 640 * paths will be resolved relative to it. 641 * 642 * @param newDir new directory. 643 */ 644 @Override 645 public void setWorkingDirectory(Path newDir) { 646 workingDir = newDir; 647 } 648 649 /** 650 * Get the current working directory for the given file system 651 * 652 * @return the directory pathname 653 */ 654 @Override 655 public Path getWorkingDirectory() { 656 if (workingDir == null) { 657 workingDir = getHomeDirectory(); 658 } 659 return workingDir; 660 } 661 662 /** 663 * Make the given file and all non-existent parents into 664 * directories. Has the semantics of Unix 'mkdir -p'. 665 * Existence of the directory hierarchy is not an error. 666 */ 667 @Override 668 public boolean mkdirs(Path f, FsPermission permission) throws IOException { 669 Map<String, String> params = new HashMap<String, String>(); 670 params.put(OP_PARAM, Operation.MKDIRS.toString()); 671 params.put(PERMISSION_PARAM, permissionToString(permission)); 672 HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(), 673 params, f, true); 674 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 675 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 676 return (Boolean) json.get(MKDIRS_JSON); 677 } 678 679 /** 680 * Return a file status object that represents the path. 681 * 682 * @param f The path we want information from 683 * 684 * @return a FileStatus object 685 * 686 * @throws FileNotFoundException when the path does not exist; 687 * IOException see specific implementation 688 */ 689 @Override 690 public FileStatus getFileStatus(Path f) throws IOException { 691 Map<String, String> params = new HashMap<String, String>(); 692 params.put(OP_PARAM, Operation.GETFILESTATUS.toString()); 693 HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(), 694 params, f, true); 695 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 696 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 697 json = (JSONObject) json.get(FILE_STATUS_JSON); 698 f = makeQualified(f); 699 return createFileStatus(f, json); 700 } 701 702 /** 703 * Return the current user's home directory in this filesystem. 704 * The default implementation returns "/user/$USER/". 705 */ 706 @Override 707 public Path getHomeDirectory() { 708 Map<String, String> params = new HashMap<String, String>(); 709 params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString()); 710 try { 711 HttpURLConnection conn = 712 getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params, 713 new Path(getUri().toString(), "/"), false); 714 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 715 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 716 return new Path((String) json.get(HOME_DIR_JSON)); 717 } catch (IOException ex) { 718 throw new RuntimeException(ex); 719 } 720 } 721 722 /** 723 * Set owner of a path (i.e. a file or a directory). 724 * The parameters username and groupname cannot both be null. 725 * 726 * @param p The path 727 * @param username If it is null, the original username remains unchanged. 728 * @param groupname If it is null, the original groupname remains unchanged. 729 */ 730 @Override 731 public void setOwner(Path p, String username, String groupname) 732 throws IOException { 733 Map<String, String> params = new HashMap<String, String>(); 734 params.put(OP_PARAM, Operation.SETOWNER.toString()); 735 params.put(OWNER_PARAM, username); 736 params.put(GROUP_PARAM, groupname); 737 HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(), 738 params, p, true); 739 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 740 } 741 742 /** 743 * Set permission of a path. 744 * 745 * @param p path. 746 * @param permission permission. 747 */ 748 @Override 749 public void setPermission(Path p, FsPermission permission) throws IOException { 750 Map<String, String> params = new HashMap<String, String>(); 751 params.put(OP_PARAM, Operation.SETPERMISSION.toString()); 752 params.put(PERMISSION_PARAM, permissionToString(permission)); 753 HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true); 754 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 755 } 756 757 /** 758 * Set access time of a file 759 * 760 * @param p The path 761 * @param mtime Set the modification time of this file. 762 * The number of milliseconds since Jan 1, 1970. 763 * A value of -1 means that this call should not set modification time. 764 * @param atime Set the access time of this file. 765 * The number of milliseconds since Jan 1, 1970. 766 * A value of -1 means that this call should not set access time. 767 */ 768 @Override 769 public void setTimes(Path p, long mtime, long atime) throws IOException { 770 Map<String, String> params = new HashMap<String, String>(); 771 params.put(OP_PARAM, Operation.SETTIMES.toString()); 772 params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime)); 773 params.put(ACCESS_TIME_PARAM, Long.toString(atime)); 774 HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(), 775 params, p, true); 776 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 777 } 778 779 /** 780 * Set replication for an existing file. 781 * 782 * @param src file name 783 * @param replication new replication 784 * 785 * @return true if successful; 786 * false if file does not exist or is a directory 787 * 788 * @throws IOException 789 */ 790 @Override 791 public boolean setReplication(Path src, short replication) 792 throws IOException { 793 Map<String, String> params = new HashMap<String, String>(); 794 params.put(OP_PARAM, Operation.SETREPLICATION.toString()); 795 params.put(REPLICATION_PARAM, Short.toString(replication)); 796 HttpURLConnection conn = 797 getConnection(Operation.SETREPLICATION.getMethod(), params, src, true); 798 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 799 JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn); 800 return (Boolean) json.get(SET_REPLICATION_JSON); 801 } 802 803 private FileStatus createFileStatus(Path parent, JSONObject json) { 804 String pathSuffix = (String) json.get(PATH_SUFFIX_JSON); 805 Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix); 806 FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON)); 807 long len = (Long) json.get(LENGTH_JSON); 808 String owner = (String) json.get(OWNER_JSON); 809 String group = (String) json.get(GROUP_JSON); 810 FsPermission permission = 811 new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8)); 812 long aTime = (Long) json.get(ACCESS_TIME_JSON); 813 long mTime = (Long) json.get(MODIFICATION_TIME_JSON); 814 long blockSize = (Long) json.get(BLOCK_SIZE_JSON); 815 short replication = ((Long) json.get(REPLICATION_JSON)).shortValue(); 816 FileStatus fileStatus = null; 817 818 switch (type) { 819 case FILE: 820 case DIRECTORY: 821 fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY), 822 replication, blockSize, mTime, aTime, 823 permission, owner, group, path); 824 break; 825 case SYMLINK: 826 Path symLink = null; 827 fileStatus = new FileStatus(len, false, 828 replication, blockSize, mTime, aTime, 829 permission, owner, group, symLink, 830 path); 831 } 832 return fileStatus; 833 } 834 835 @Override 836 public ContentSummary getContentSummary(Path f) throws IOException { 837 Map<String, String> params = new HashMap<String, String>(); 838 params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString()); 839 HttpURLConnection conn = 840 getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true); 841 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 842 JSONObject json = (JSONObject) ((JSONObject) 843 HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON); 844 return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON), 845 (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON), 846 (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON), 847 (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON), 848 (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON), 849 (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON) 850 ); 851 } 852 853 @Override 854 public FileChecksum getFileChecksum(Path f) throws IOException { 855 Map<String, String> params = new HashMap<String, String>(); 856 params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString()); 857 HttpURLConnection conn = 858 getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true); 859 HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); 860 final JSONObject json = (JSONObject) ((JSONObject) 861 HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON); 862 return new FileChecksum() { 863 @Override 864 public String getAlgorithmName() { 865 return (String) json.get(CHECKSUM_ALGORITHM_JSON); 866 } 867 868 @Override 869 public int getLength() { 870 return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue(); 871 } 872 873 @Override 874 public byte[] getBytes() { 875 return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON)); 876 } 877 878 @Override 879 public void write(DataOutput out) throws IOException { 880 throw new UnsupportedOperationException(); 881 } 882 883 @Override 884 public void readFields(DataInput in) throws IOException { 885 throw new UnsupportedOperationException(); 886 } 887 }; 888 } 889 890 891 @Override 892 public Token<?> getDelegationToken(final String renewer) 893 throws IOException { 894 return doAsRealUserIfNecessary(new Callable<Token<?>>() { 895 @Override 896 public Token<?> call() throws Exception { 897 return HttpFSKerberosAuthenticator. 898 getDelegationToken(uri, httpFSAddr, authToken, renewer); 899 } 900 }); 901 } 902 903 public long renewDelegationToken(final Token<?> token) throws IOException { 904 return doAsRealUserIfNecessary(new Callable<Long>() { 905 @Override 906 public Long call() throws Exception { 907 return HttpFSKerberosAuthenticator. 908 renewDelegationToken(uri, authToken, token); 909 } 910 }); 911 } 912 913 public void cancelDelegationToken(final Token<?> token) throws IOException { 914 HttpFSKerberosAuthenticator. 915 cancelDelegationToken(uri, authToken, token); 916 } 917 918 @Override 919 public Token<?> getRenewToken() { 920 return delegationToken; 921 } 922 923 @Override 924 public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) { 925 delegationToken = token; 926 } 927 928 }