001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.http.client;
019    
020    import java.util.ArrayList;
021    import java.util.List;
022    import org.apache.hadoop.classification.InterfaceAudience;
023    import org.apache.hadoop.conf.Configuration;
024    import org.apache.hadoop.fs.ContentSummary;
025    import org.apache.hadoop.fs.DelegationTokenRenewer;
026    import org.apache.hadoop.fs.FSDataInputStream;
027    import org.apache.hadoop.fs.FSDataOutputStream;
028    import org.apache.hadoop.fs.FileChecksum;
029    import org.apache.hadoop.fs.FileStatus;
030    import org.apache.hadoop.fs.FileSystem;
031    import org.apache.hadoop.fs.Path;
032    import org.apache.hadoop.fs.PositionedReadable;
033    import org.apache.hadoop.fs.Seekable;
034    import org.apache.hadoop.fs.permission.FsPermission;
035    import org.apache.hadoop.hdfs.DFSConfigKeys;
036    import org.apache.hadoop.net.NetUtils;
037    import org.apache.hadoop.security.UserGroupInformation;
038    import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
039    import org.apache.hadoop.security.authentication.client.Authenticator;
040    import org.apache.hadoop.security.token.Token;
041    import org.apache.hadoop.security.token.TokenIdentifier;
042    import org.apache.hadoop.util.Progressable;
043    import org.apache.hadoop.util.ReflectionUtils;
044    import org.apache.hadoop.util.StringUtils;
045    import org.json.simple.JSONArray;
046    import org.json.simple.JSONObject;
047    
048    import java.io.BufferedInputStream;
049    import java.io.BufferedOutputStream;
050    import java.io.DataInput;
051    import java.io.DataOutput;
052    import java.io.FileNotFoundException;
053    import java.io.FilterInputStream;
054    import java.io.IOException;
055    import java.io.InputStream;
056    import java.io.OutputStream;
057    import java.net.HttpURLConnection;
058    import java.net.InetSocketAddress;
059    import java.net.URI;
060    import java.net.URISyntaxException;
061    import java.net.URL;
062    import java.security.PrivilegedExceptionAction;
063    import java.text.MessageFormat;
064    import java.util.HashMap;
065    import java.util.List;
066    import java.util.Map;
067    import java.util.concurrent.Callable;
068    
069    /**
070     * HttpFSServer implementation of the FileSystemAccess FileSystem.
071     * <p/>
072     * This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
073     */
074    @InterfaceAudience.Private
075    public class HttpFSFileSystem extends FileSystem
076      implements DelegationTokenRenewer.Renewable {
077    
078      public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME;
079    
080      public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION;
081    
082      public static final String SCHEME = "webhdfs";
083    
084      public static final String OP_PARAM = "op";
085      public static final String DO_AS_PARAM = "doas";
086      public static final String OVERWRITE_PARAM = "overwrite";
087      public static final String REPLICATION_PARAM = "replication";
088      public static final String BLOCKSIZE_PARAM = "blocksize";
089      public static final String PERMISSION_PARAM = "permission";
090      public static final String DESTINATION_PARAM = "destination";
091      public static final String RECURSIVE_PARAM = "recursive";
092      public static final String SOURCES_PARAM = "sources";
093      public static final String OWNER_PARAM = "owner";
094      public static final String GROUP_PARAM = "group";
095      public static final String MODIFICATION_TIME_PARAM = "modificationtime";
096      public static final String ACCESS_TIME_PARAM = "accesstime";
097    
098      public static final Short DEFAULT_PERMISSION = 0755;
099    
100      public static final String RENAME_JSON = "boolean";
101    
102      public static final String DELETE_JSON = "boolean";
103    
104      public static final String MKDIRS_JSON = "boolean";
105    
106      public static final String HOME_DIR_JSON = "Path";
107    
108      public static final String SET_REPLICATION_JSON = "boolean";
109    
110      public static final String UPLOAD_CONTENT_TYPE= "application/octet-stream";
111    
112      public static enum FILE_TYPE {
113        FILE, DIRECTORY, SYMLINK;
114    
115        public static FILE_TYPE getType(FileStatus fileStatus) {
116          if (fileStatus.isFile()) {
117            return FILE;
118          }
119          if (fileStatus.isDirectory()) {
120            return DIRECTORY;
121          }
122          if (fileStatus.isSymlink()) {
123            return SYMLINK;
124          }
125          throw new IllegalArgumentException("Could not determine filetype for: " +
126                                             fileStatus.getPath());
127        }
128      }
129    
130      public static final String FILE_STATUSES_JSON = "FileStatuses";
131      public static final String FILE_STATUS_JSON = "FileStatus";
132      public static final String PATH_SUFFIX_JSON = "pathSuffix";
133      public static final String TYPE_JSON = "type";
134      public static final String LENGTH_JSON = "length";
135      public static final String OWNER_JSON = "owner";
136      public static final String GROUP_JSON = "group";
137      public static final String PERMISSION_JSON = "permission";
138      public static final String ACCESS_TIME_JSON = "accessTime";
139      public static final String MODIFICATION_TIME_JSON = "modificationTime";
140      public static final String BLOCK_SIZE_JSON = "blockSize";
141      public static final String REPLICATION_JSON = "replication";
142    
143      public static final String FILE_CHECKSUM_JSON = "FileChecksum";
144      public static final String CHECKSUM_ALGORITHM_JSON = "algorithm";
145      public static final String CHECKSUM_BYTES_JSON = "bytes";
146      public static final String CHECKSUM_LENGTH_JSON = "length";
147    
148      public static final String CONTENT_SUMMARY_JSON = "ContentSummary";
149      public static final String CONTENT_SUMMARY_DIRECTORY_COUNT_JSON = "directoryCount";
150      public static final String CONTENT_SUMMARY_FILE_COUNT_JSON = "fileCount";
151      public static final String CONTENT_SUMMARY_LENGTH_JSON = "length";
152      public static final String CONTENT_SUMMARY_QUOTA_JSON = "quota";
153      public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed";
154      public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota";
155    
156      public static final String ERROR_JSON = "RemoteException";
157      public static final String ERROR_EXCEPTION_JSON = "exception";
158      public static final String ERROR_CLASSNAME_JSON = "javaClassName";
159      public static final String ERROR_MESSAGE_JSON = "message";
160    
161      public static final int HTTP_TEMPORARY_REDIRECT = 307;
162    
163      private static final String HTTP_GET = "GET";
164      private static final String HTTP_PUT = "PUT";
165      private static final String HTTP_POST = "POST";
166      private static final String HTTP_DELETE = "DELETE";
167    
168      @InterfaceAudience.Private
169      public static enum Operation {
170        OPEN(HTTP_GET), GETFILESTATUS(HTTP_GET), LISTSTATUS(HTTP_GET),
171        GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET),
172        GETFILECHECKSUM(HTTP_GET),  GETFILEBLOCKLOCATIONS(HTTP_GET),
173        INSTRUMENTATION(HTTP_GET),
174        APPEND(HTTP_POST), CONCAT(HTTP_POST),
175        CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT),
176        SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT),
177        DELETE(HTTP_DELETE);
178    
179        private String httpMethod;
180    
181        Operation(String httpMethod) {
182          this.httpMethod = httpMethod;
183        }
184    
185        public String getMethod() {
186          return httpMethod;
187        }
188    
189      }
190    
191    
192      private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
193      private URI uri;
194      private InetSocketAddress httpFSAddr;
195      private Path workingDir;
196      private UserGroupInformation realUser;
197      private String doAs;
198      private Token<?> delegationToken;
199    
200      //This method enables handling UGI doAs with SPNEGO, we have to
201      //fallback to the realuser who logged in with Kerberos credentials
202      private <T> T doAsRealUserIfNecessary(final Callable<T> callable)
203        throws IOException {
204        try {
205          if (realUser.getShortUserName().equals(doAs)) {
206            return callable.call();
207          } else {
208            return realUser.doAs(new PrivilegedExceptionAction<T>() {
209              @Override
210              public T run() throws Exception {
211                return callable.call();
212              }
213            });
214          }
215        } catch (Exception ex) {
216          throw new IOException(ex.toString(), ex);
217        }
218      }
219    
220      /**
221       * Convenience method that creates a <code>HttpURLConnection</code> for the
222       * HttpFSServer file system operations.
223       * <p/>
224       * This methods performs and injects any needed authentication credentials
225       * via the {@link #getConnection(URL, String)} method
226       *
227       * @param method the HTTP method.
228       * @param params the query string parameters.
229       * @param path the file path
230       * @param makeQualified if the path should be 'makeQualified'
231       *
232       * @return a <code>HttpURLConnection</code> for the HttpFSServer server,
233       *         authenticated and ready to use for the specified path and file system operation.
234       *
235       * @throws IOException thrown if an IO error occurrs.
236       */
237      private HttpURLConnection getConnection(final String method,
238          Map<String, String> params, Path path, boolean makeQualified)
239          throws IOException {
240        if (!realUser.getShortUserName().equals(doAs)) {
241          params.put(DO_AS_PARAM, doAs);
242        }
243        HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken);
244        if (makeQualified) {
245          path = makeQualified(path);
246        }
247        final URL url = HttpFSUtils.createHttpURL(path, params);
248        return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() {
249          @Override
250          public HttpURLConnection call() throws Exception {
251            return getConnection(url, method);
252          }
253        });
254      }
255    
256      /**
257       * Convenience method that creates a <code>HttpURLConnection</code> for the specified URL.
258       * <p/>
259       * This methods performs and injects any needed authentication credentials.
260       *
261       * @param url url to connect to.
262       * @param method the HTTP method.
263       *
264       * @return a <code>HttpURLConnection</code> for the HttpFSServer server, authenticated and ready to use for
265       *         the specified path and file system operation.
266       *
267       * @throws IOException thrown if an IO error occurrs.
268       */
269      private HttpURLConnection getConnection(URL url, String method) throws IOException {
270        Class<? extends Authenticator> klass =
271          getConf().getClass("httpfs.authenticator.class",
272                             HttpFSKerberosAuthenticator.class, Authenticator.class);
273        Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf());
274        try {
275          HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken);
276          conn.setRequestMethod(method);
277          if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) {
278            conn.setDoOutput(true);
279          }
280          return conn;
281        } catch (Exception ex) {
282          throw new IOException(ex);
283        }
284      }
285    
286      /**
287       * Called after a new FileSystem instance is constructed.
288       *
289       * @param name a uri whose authority section names the host, port, etc. for this FileSystem
290       * @param conf the configuration
291       */
292      @Override
293      public void initialize(URI name, Configuration conf) throws IOException {
294        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
295    
296        //the real use is the one that has the Kerberos credentials needed for
297        //SPNEGO to work
298        realUser = ugi.getRealUser();
299        if (realUser == null) {
300          realUser = UserGroupInformation.getLoginUser();
301        }
302        doAs = ugi.getShortUserName();
303        super.initialize(name, conf);
304        try {
305          uri = new URI(name.getScheme() + "://" + name.getAuthority());
306          httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString());
307        } catch (URISyntaxException ex) {
308          throw new IOException(ex);
309        }
310      }
311    
312      @Override
313      public String getScheme() {
314        return SCHEME;
315      }
316    
317      /**
318       * Returns a URI whose scheme and authority identify this FileSystem.
319       *
320       * @return the URI whose scheme and authority identify this FileSystem.
321       */
322      @Override
323      public URI getUri() {
324        return uri;
325      }
326    
327      /**
328       * Get the default port for this file system.
329       * @return the default port or 0 if there isn't one
330       */
331      @Override
332      protected int getDefaultPort() {
333        return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
334            DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
335      }
336    
337      /**
338       * HttpFSServer subclass of the <code>FSDataInputStream</code>.
339       * <p/>
340       * This implementation does not support the
341       * <code>PositionReadable</code> and <code>Seekable</code> methods.
342       */
343      private static class HttpFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable {
344    
345        protected HttpFSDataInputStream(InputStream in, int bufferSize) {
346          super(new BufferedInputStream(in, bufferSize));
347        }
348    
349        @Override
350        public int read(long position, byte[] buffer, int offset, int length) throws IOException {
351          throw new UnsupportedOperationException();
352        }
353    
354        @Override
355        public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
356          throw new UnsupportedOperationException();
357        }
358    
359        @Override
360        public void readFully(long position, byte[] buffer) throws IOException {
361          throw new UnsupportedOperationException();
362        }
363    
364        @Override
365        public void seek(long pos) throws IOException {
366          throw new UnsupportedOperationException();
367        }
368    
369        @Override
370        public long getPos() throws IOException {
371          throw new UnsupportedOperationException();
372        }
373    
374        @Override
375        public boolean seekToNewSource(long targetPos) throws IOException {
376          throw new UnsupportedOperationException();
377        }
378      }
379    
380      /**
381       * Opens an FSDataInputStream at the indicated Path.
382       * </p>
383       * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the
384       * <code>PositionReadable</code> and <code>Seekable</code> methods.
385       *
386       * @param f the file name to open
387       * @param bufferSize the size of the buffer to be used.
388       */
389      @Override
390      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
391        Map<String, String> params = new HashMap<String, String>();
392        params.put(OP_PARAM, Operation.OPEN.toString());
393        HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params,
394                                               f, true);
395        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
396        return new FSDataInputStream(
397          new HttpFSDataInputStream(conn.getInputStream(), bufferSize));
398      }
399    
400      /**
401       * HttpFSServer subclass of the <code>FSDataOutputStream</code>.
402       * <p/>
403       * This implementation closes the underlying HTTP connection validating the Http connection status
404       * at closing time.
405       */
406      private static class HttpFSDataOutputStream extends FSDataOutputStream {
407        private HttpURLConnection conn;
408        private int closeStatus;
409    
410        public HttpFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats)
411          throws IOException {
412          super(out, stats);
413          this.conn = conn;
414          this.closeStatus = closeStatus;
415        }
416    
417        @Override
418        public void close() throws IOException {
419          try {
420            super.close();
421          } finally {
422            HttpFSUtils.validateResponse(conn, closeStatus);
423          }
424        }
425    
426      }
427    
428      /**
429       * Converts a <code>FsPermission</code> to a Unix octal representation.
430       *
431       * @param p the permission.
432       *
433       * @return the Unix string symbolic reprentation.
434       */
435      public static String permissionToString(FsPermission p) {
436        return  Integer.toString((p == null) ? DEFAULT_PERMISSION : p.toShort(), 8);
437      }
438    
439      /*
440       * Common handling for uploading data for create and append operations.
441       */
442      private FSDataOutputStream uploadData(String method, Path f, Map<String, String> params,
443                                            int bufferSize, int expectedStatus) throws IOException {
444        HttpURLConnection conn = getConnection(method, params, f, true);
445        conn.setInstanceFollowRedirects(false);
446        boolean exceptionAlreadyHandled = false;
447        try {
448          if (conn.getResponseCode() == HTTP_TEMPORARY_REDIRECT) {
449            exceptionAlreadyHandled = true;
450            String location = conn.getHeaderField("Location");
451            if (location != null) {
452              conn = getConnection(new URL(location), method);
453              conn.setRequestProperty("Content-Type", UPLOAD_CONTENT_TYPE);
454              try {
455                OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize);
456                return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics);
457              } catch (IOException ex) {
458                HttpFSUtils.validateResponse(conn, expectedStatus);
459                throw ex;
460              }
461            } else {
462              HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
463              throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]");
464            }
465          } else {
466            throw new IOException(
467              MessageFormat.format("Expected HTTP status was [307], received [{0}]",
468                                   conn.getResponseCode()));
469          }
470        } catch (IOException ex) {
471          if (exceptionAlreadyHandled) {
472            throw ex;
473          } else {
474            HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
475            throw ex;
476          }
477        }
478      }
479    
480    
481      /**
482       * Opens an FSDataOutputStream at the indicated Path with write-progress
483       * reporting.
484       * <p/>
485       * IMPORTANT: The <code>Progressable</code> parameter is not used.
486       *
487       * @param f the file name to open.
488       * @param permission file permission.
489       * @param overwrite if a file with this name already exists, then if true,
490       * the file will be overwritten, and if false an error will be thrown.
491       * @param bufferSize the size of the buffer to be used.
492       * @param replication required block replication for the file.
493       * @param blockSize block size.
494       * @param progress progressable.
495       *
496       * @throws IOException
497       * @see #setPermission(Path, FsPermission)
498       */
499      @Override
500      public FSDataOutputStream create(Path f, FsPermission permission,
501                                       boolean overwrite, int bufferSize,
502                                       short replication, long blockSize,
503                                       Progressable progress) throws IOException {
504        Map<String, String> params = new HashMap<String, String>();
505        params.put(OP_PARAM, Operation.CREATE.toString());
506        params.put(OVERWRITE_PARAM, Boolean.toString(overwrite));
507        params.put(REPLICATION_PARAM, Short.toString(replication));
508        params.put(BLOCKSIZE_PARAM, Long.toString(blockSize));
509        params.put(PERMISSION_PARAM, permissionToString(permission));
510        return uploadData(Operation.CREATE.getMethod(), f, params, bufferSize,
511                          HttpURLConnection.HTTP_CREATED);
512      }
513    
514    
515      /**
516       * Append to an existing file (optional operation).
517       * <p/>
518       * IMPORTANT: The <code>Progressable</code> parameter is not used.
519       *
520       * @param f the existing file to be appended.
521       * @param bufferSize the size of the buffer to be used.
522       * @param progress for reporting progress if it is not null.
523       *
524       * @throws IOException
525       */
526      @Override
527      public FSDataOutputStream append(Path f, int bufferSize,
528                                       Progressable progress) throws IOException {
529        Map<String, String> params = new HashMap<String, String>();
530        params.put(OP_PARAM, Operation.APPEND.toString());
531        return uploadData(Operation.APPEND.getMethod(), f, params, bufferSize,
532                          HttpURLConnection.HTTP_OK);
533      }
534    
535      /**
536       * Concat existing files together.
537       * @param f the path to the target destination.
538       * @param psrcs the paths to the sources to use for the concatenation.
539       *
540       * @throws IOException
541       */
542      @Override
543      public void concat(Path f, Path[] psrcs) throws IOException {
544        List<String> strPaths = new ArrayList<String>(psrcs.length);
545        for(Path psrc : psrcs) {
546          strPaths.add(psrc.toUri().getPath());
547        }
548        String srcs = StringUtils.join(",", strPaths);
549    
550        Map<String, String> params = new HashMap<String, String>();
551        params.put(OP_PARAM, Operation.CONCAT.toString());
552        params.put(SOURCES_PARAM, srcs);
553        HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
554            params, f, true);
555        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
556      }
557    
558      /**
559       * Renames Path src to Path dst.  Can take place on local fs
560       * or remote DFS.
561       */
562      @Override
563      public boolean rename(Path src, Path dst) throws IOException {
564        Map<String, String> params = new HashMap<String, String>();
565        params.put(OP_PARAM, Operation.RENAME.toString());
566        params.put(DESTINATION_PARAM, dst.toString());
567        HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(),
568                                               params, src, true);
569        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
570        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
571        return (Boolean) json.get(RENAME_JSON);
572      }
573    
574      /**
575       * Delete a file.
576       *
577       * @deprecated Use delete(Path, boolean) instead
578       */
579      @SuppressWarnings({"deprecation"})
580      @Deprecated
581      @Override
582      public boolean delete(Path f) throws IOException {
583        return delete(f, false);
584      }
585    
586      /**
587       * Delete a file.
588       *
589       * @param f the path to delete.
590       * @param recursive if path is a directory and set to
591       * true, the directory is deleted else throws an exception. In
592       * case of a file the recursive can be set to either true or false.
593       *
594       * @return true if delete is successful else false.
595       *
596       * @throws IOException
597       */
598      @Override
599      public boolean delete(Path f, boolean recursive) throws IOException {
600        Map<String, String> params = new HashMap<String, String>();
601        params.put(OP_PARAM, Operation.DELETE.toString());
602        params.put(RECURSIVE_PARAM, Boolean.toString(recursive));
603        HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(),
604                                               params, f, true);
605        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
606        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
607        return (Boolean) json.get(DELETE_JSON);
608      }
609    
610      /**
611       * List the statuses of the files/directories in the given path if the path is
612       * a directory.
613       *
614       * @param f given path
615       *
616       * @return the statuses of the files/directories in the given patch
617       *
618       * @throws IOException
619       */
620      @Override
621      public FileStatus[] listStatus(Path f) throws IOException {
622        Map<String, String> params = new HashMap<String, String>();
623        params.put(OP_PARAM, Operation.LISTSTATUS.toString());
624        HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(),
625                                               params, f, true);
626        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
627        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
628        json = (JSONObject) json.get(FILE_STATUSES_JSON);
629        JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON);
630        FileStatus[] array = new FileStatus[jsonArray.size()];
631        f = makeQualified(f);
632        for (int i = 0; i < jsonArray.size(); i++) {
633          array[i] = createFileStatus(f, (JSONObject) jsonArray.get(i));
634        }
635        return array;
636      }
637    
638      /**
639       * Set the current working directory for the given file system. All relative
640       * paths will be resolved relative to it.
641       *
642       * @param newDir new directory.
643       */
644      @Override
645      public void setWorkingDirectory(Path newDir) {
646        workingDir = newDir;
647      }
648    
649      /**
650       * Get the current working directory for the given file system
651       *
652       * @return the directory pathname
653       */
654      @Override
655      public Path getWorkingDirectory() {
656        if (workingDir == null) {
657          workingDir = getHomeDirectory();
658        }
659        return workingDir;
660      }
661    
662      /**
663       * Make the given file and all non-existent parents into
664       * directories. Has the semantics of Unix 'mkdir -p'.
665       * Existence of the directory hierarchy is not an error.
666       */
667      @Override
668      public boolean mkdirs(Path f, FsPermission permission) throws IOException {
669        Map<String, String> params = new HashMap<String, String>();
670        params.put(OP_PARAM, Operation.MKDIRS.toString());
671        params.put(PERMISSION_PARAM, permissionToString(permission));
672        HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(),
673                                               params, f, true);
674        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
675        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
676        return (Boolean) json.get(MKDIRS_JSON);
677      }
678    
679      /**
680       * Return a file status object that represents the path.
681       *
682       * @param f The path we want information from
683       *
684       * @return a FileStatus object
685       *
686       * @throws FileNotFoundException when the path does not exist;
687       * IOException see specific implementation
688       */
689      @Override
690      public FileStatus getFileStatus(Path f) throws IOException {
691        Map<String, String> params = new HashMap<String, String>();
692        params.put(OP_PARAM, Operation.GETFILESTATUS.toString());
693        HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(),
694                                               params, f, true);
695        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
696        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
697        json = (JSONObject) json.get(FILE_STATUS_JSON);
698        f = makeQualified(f);
699        return createFileStatus(f, json);
700      }
701    
702      /**
703       * Return the current user's home directory in this filesystem.
704       * The default implementation returns "/user/$USER/".
705       */
706      @Override
707      public Path getHomeDirectory() {
708        Map<String, String> params = new HashMap<String, String>();
709        params.put(OP_PARAM, Operation.GETHOMEDIRECTORY.toString());
710        try {
711          HttpURLConnection conn =
712            getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params,
713                          new Path(getUri().toString(), "/"), false);
714          HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
715          JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
716          return new Path((String) json.get(HOME_DIR_JSON));
717        } catch (IOException ex) {
718          throw new RuntimeException(ex);
719        }
720      }
721    
722      /**
723       * Set owner of a path (i.e. a file or a directory).
724       * The parameters username and groupname cannot both be null.
725       *
726       * @param p The path
727       * @param username If it is null, the original username remains unchanged.
728       * @param groupname If it is null, the original groupname remains unchanged.
729       */
730      @Override
731      public void setOwner(Path p, String username, String groupname)
732        throws IOException {
733        Map<String, String> params = new HashMap<String, String>();
734        params.put(OP_PARAM, Operation.SETOWNER.toString());
735        params.put(OWNER_PARAM, username);
736        params.put(GROUP_PARAM, groupname);
737        HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(),
738                                               params, p, true);
739        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
740      }
741    
742      /**
743       * Set permission of a path.
744       *
745       * @param p path.
746       * @param permission permission.
747       */
748      @Override
749      public void setPermission(Path p, FsPermission permission) throws IOException {
750        Map<String, String> params = new HashMap<String, String>();
751        params.put(OP_PARAM, Operation.SETPERMISSION.toString());
752        params.put(PERMISSION_PARAM, permissionToString(permission));
753        HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true);
754        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
755      }
756    
757      /**
758       * Set access time of a file
759       *
760       * @param p The path
761       * @param mtime Set the modification time of this file.
762       * The number of milliseconds since Jan 1, 1970.
763       * A value of -1 means that this call should not set modification time.
764       * @param atime Set the access time of this file.
765       * The number of milliseconds since Jan 1, 1970.
766       * A value of -1 means that this call should not set access time.
767       */
768      @Override
769      public void setTimes(Path p, long mtime, long atime) throws IOException {
770        Map<String, String> params = new HashMap<String, String>();
771        params.put(OP_PARAM, Operation.SETTIMES.toString());
772        params.put(MODIFICATION_TIME_PARAM, Long.toString(mtime));
773        params.put(ACCESS_TIME_PARAM, Long.toString(atime));
774        HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(),
775                                               params, p, true);
776        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
777      }
778    
779      /**
780       * Set replication for an existing file.
781       *
782       * @param src file name
783       * @param replication new replication
784       *
785       * @return true if successful;
786       *         false if file does not exist or is a directory
787       *
788       * @throws IOException
789       */
790      @Override
791      public boolean setReplication(Path src, short replication)
792        throws IOException {
793        Map<String, String> params = new HashMap<String, String>();
794        params.put(OP_PARAM, Operation.SETREPLICATION.toString());
795        params.put(REPLICATION_PARAM, Short.toString(replication));
796        HttpURLConnection conn =
797          getConnection(Operation.SETREPLICATION.getMethod(), params, src, true);
798        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
799        JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
800        return (Boolean) json.get(SET_REPLICATION_JSON);
801      }
802    
803      private FileStatus createFileStatus(Path parent, JSONObject json) {
804        String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
805        Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
806        FILE_TYPE type = FILE_TYPE.valueOf((String) json.get(TYPE_JSON));
807        long len = (Long) json.get(LENGTH_JSON);
808        String owner = (String) json.get(OWNER_JSON);
809        String group = (String) json.get(GROUP_JSON);
810        FsPermission permission =
811          new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8));
812        long aTime = (Long) json.get(ACCESS_TIME_JSON);
813        long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
814        long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
815        short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
816        FileStatus fileStatus = null;
817    
818        switch (type) {
819          case FILE:
820          case DIRECTORY:
821            fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
822                                        replication, blockSize, mTime, aTime,
823                                        permission, owner, group, path);
824            break;
825          case SYMLINK:
826            Path symLink = null;
827            fileStatus = new FileStatus(len, false,
828                                        replication, blockSize, mTime, aTime,
829                                        permission, owner, group, symLink,
830                                        path);
831        }
832        return fileStatus;
833      }
834    
835      @Override
836      public ContentSummary getContentSummary(Path f) throws IOException {
837        Map<String, String> params = new HashMap<String, String>();
838        params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString());
839        HttpURLConnection conn =
840          getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true);
841        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
842        JSONObject json = (JSONObject) ((JSONObject)
843          HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
844        return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON),
845                                  (Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON),
846                                  (Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON),
847                                  (Long) json.get(CONTENT_SUMMARY_QUOTA_JSON),
848                                  (Long) json.get(CONTENT_SUMMARY_SPACE_CONSUMED_JSON),
849                                  (Long) json.get(CONTENT_SUMMARY_SPACE_QUOTA_JSON)
850        );
851      }
852    
853      @Override
854      public FileChecksum getFileChecksum(Path f) throws IOException {
855        Map<String, String> params = new HashMap<String, String>();
856        params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString());
857        HttpURLConnection conn =
858          getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true);
859        HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
860        final JSONObject json = (JSONObject) ((JSONObject)
861          HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON);
862        return new FileChecksum() {
863          @Override
864          public String getAlgorithmName() {
865            return (String) json.get(CHECKSUM_ALGORITHM_JSON);
866          }
867    
868          @Override
869          public int getLength() {
870            return ((Long) json.get(CHECKSUM_LENGTH_JSON)).intValue();
871          }
872    
873          @Override
874          public byte[] getBytes() {
875            return StringUtils.hexStringToByte((String) json.get(CHECKSUM_BYTES_JSON));
876          }
877    
878          @Override
879          public void write(DataOutput out) throws IOException {
880            throw new UnsupportedOperationException();
881          }
882    
883          @Override
884          public void readFields(DataInput in) throws IOException {
885            throw new UnsupportedOperationException();
886          }
887        };
888      }
889    
890    
891      @Override
892      public Token<?> getDelegationToken(final String renewer)
893        throws IOException {
894        return doAsRealUserIfNecessary(new Callable<Token<?>>() {
895          @Override
896          public Token<?> call() throws Exception {
897            return HttpFSKerberosAuthenticator.
898              getDelegationToken(uri, httpFSAddr, authToken, renewer);
899          }
900        });
901      }
902    
903      public long renewDelegationToken(final Token<?> token) throws IOException {
904        return doAsRealUserIfNecessary(new Callable<Long>() {
905          @Override
906          public Long call() throws Exception {
907            return HttpFSKerberosAuthenticator.
908              renewDelegationToken(uri,  authToken, token);
909          }
910        });
911      }
912    
913      public void cancelDelegationToken(final Token<?> token) throws IOException {
914        HttpFSKerberosAuthenticator.
915          cancelDelegationToken(uri, authToken, token);
916      }
917    
918      @Override
919      public Token<?> getRenewToken() {
920        return delegationToken;
921      }
922    
923      @Override
924      public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
925        delegationToken = token;
926      }
927    
928    }