001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.fs;
020    
021    import java.io.BufferedOutputStream;
022    import java.io.DataOutput;
023    import java.io.File;
024    import java.io.FileInputStream;
025    import java.io.FileNotFoundException;
026    import java.io.FileOutputStream;
027    import java.io.IOException;
028    import java.io.OutputStream;
029    import java.io.FileDescriptor;
030    import java.net.URI;
031    import java.nio.ByteBuffer;
032    import java.util.Arrays;
033    import java.util.EnumSet;
034    import java.util.StringTokenizer;
035    
036    import org.apache.hadoop.classification.InterfaceAudience;
037    import org.apache.hadoop.classification.InterfaceStability;
038    import org.apache.hadoop.conf.Configuration;
039    import org.apache.hadoop.fs.permission.FsPermission;
040    import org.apache.hadoop.io.nativeio.NativeIO;
041    import org.apache.hadoop.util.Progressable;
042    import org.apache.hadoop.util.Shell;
043    import org.apache.hadoop.util.StringUtils;
044    
045    /****************************************************************
046     * Implement the FileSystem API for the raw local filesystem.
047     *
048     *****************************************************************/
049    @InterfaceAudience.Public
050    @InterfaceStability.Stable
051    public class RawLocalFileSystem extends FileSystem {
052      static final URI NAME = URI.create("file:///");
053      private Path workingDir;
054      
055      public RawLocalFileSystem() {
056        workingDir = getInitialWorkingDirectory();
057      }
058      
059      private Path makeAbsolute(Path f) {
060        if (f.isAbsolute()) {
061          return f;
062        } else {
063          return new Path(workingDir, f);
064        }
065      }
066      
067      /** Convert a path to a File. */
068      public File pathToFile(Path path) {
069        checkPath(path);
070        if (!path.isAbsolute()) {
071          path = new Path(getWorkingDirectory(), path);
072        }
073        return new File(path.toUri().getPath());
074      }
075    
076      public URI getUri() { return NAME; }
077      
078      public void initialize(URI uri, Configuration conf) throws IOException {
079        super.initialize(uri, conf);
080        setConf(conf);
081      }
082      
083      class TrackingFileInputStream extends FileInputStream {
084        public TrackingFileInputStream(File f) throws IOException {
085          super(f);
086        }
087        
088        public int read() throws IOException {
089          int result = super.read();
090          if (result != -1) {
091            statistics.incrementBytesRead(1);
092          }
093          return result;
094        }
095        
096        public int read(byte[] data) throws IOException {
097          int result = super.read(data);
098          if (result != -1) {
099            statistics.incrementBytesRead(result);
100          }
101          return result;
102        }
103        
104        public int read(byte[] data, int offset, int length) throws IOException {
105          int result = super.read(data, offset, length);
106          if (result != -1) {
107            statistics.incrementBytesRead(result);
108          }
109          return result;
110        }
111      }
112    
113      /*******************************************************
114       * For open()'s FSInputStream.
115       *******************************************************/
116      class LocalFSFileInputStream extends FSInputStream implements HasFileDescriptor {
117        private FileInputStream fis;
118        private long position;
119    
120        public LocalFSFileInputStream(Path f) throws IOException {
121          this.fis = new TrackingFileInputStream(pathToFile(f));
122        }
123        
124        public void seek(long pos) throws IOException {
125          fis.getChannel().position(pos);
126          this.position = pos;
127        }
128        
129        public long getPos() throws IOException {
130          return this.position;
131        }
132        
133        public boolean seekToNewSource(long targetPos) throws IOException {
134          return false;
135        }
136        
137        /*
138         * Just forward to the fis
139         */
140        public int available() throws IOException { return fis.available(); }
141        public void close() throws IOException { fis.close(); }
142        @Override
143        public boolean markSupported() { return false; }
144        
145        public int read() throws IOException {
146          try {
147            int value = fis.read();
148            if (value >= 0) {
149              this.position++;
150            }
151            return value;
152          } catch (IOException e) {                 // unexpected exception
153            throw new FSError(e);                   // assume native fs error
154          }
155        }
156        
157        public int read(byte[] b, int off, int len) throws IOException {
158          try {
159            int value = fis.read(b, off, len);
160            if (value > 0) {
161              this.position += value;
162            }
163            return value;
164          } catch (IOException e) {                 // unexpected exception
165            throw new FSError(e);                   // assume native fs error
166          }
167        }
168        
169        public int read(long position, byte[] b, int off, int len)
170          throws IOException {
171          ByteBuffer bb = ByteBuffer.wrap(b, off, len);
172          try {
173            return fis.getChannel().read(bb, position);
174          } catch (IOException e) {
175            throw new FSError(e);
176          }
177        }
178        
179        public long skip(long n) throws IOException {
180          long value = fis.skip(n);
181          if (value > 0) {
182            this.position += value;
183          }
184          return value;
185        }
186    
187        @Override
188        public FileDescriptor getFileDescriptor() throws IOException {
189          return fis.getFD();
190        }
191      }
192      
193      public FSDataInputStream open(Path f, int bufferSize) throws IOException {
194        if (!exists(f)) {
195          throw new FileNotFoundException(f.toString());
196        }
197        return new FSDataInputStream(new BufferedFSInputStream(
198            new LocalFSFileInputStream(f), bufferSize));
199      }
200      
201      /*********************************************************
202       * For create()'s FSOutputStream.
203       *********************************************************/
204      class LocalFSFileOutputStream extends OutputStream {
205        private FileOutputStream fos;
206        
207        private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
208          this.fos = new FileOutputStream(pathToFile(f), append);
209        }
210        
211        /*
212         * Just forward to the fos
213         */
214        public void close() throws IOException { fos.close(); }
215        public void flush() throws IOException { fos.flush(); }
216        public void write(byte[] b, int off, int len) throws IOException {
217          try {
218            fos.write(b, off, len);
219          } catch (IOException e) {                // unexpected exception
220            throw new FSError(e);                  // assume native fs error
221          }
222        }
223        
224        public void write(int b) throws IOException {
225          try {
226            fos.write(b);
227          } catch (IOException e) {              // unexpected exception
228            throw new FSError(e);                // assume native fs error
229          }
230        }
231      }
232    
233      /** {@inheritDoc} */
234      public FSDataOutputStream append(Path f, int bufferSize,
235          Progressable progress) throws IOException {
236        if (!exists(f)) {
237          throw new FileNotFoundException("File " + f + " not found");
238        }
239        if (getFileStatus(f).isDirectory()) {
240          throw new IOException("Cannot append to a diretory (=" + f + " )");
241        }
242        return new FSDataOutputStream(new BufferedOutputStream(
243            new LocalFSFileOutputStream(f, true), bufferSize), statistics);
244      }
245    
246      /** {@inheritDoc} */
247      @Override
248      public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
249        short replication, long blockSize, Progressable progress)
250        throws IOException {
251        return create(f, overwrite, true, bufferSize, replication, blockSize, progress);
252      }
253    
254      private FSDataOutputStream create(Path f, boolean overwrite,
255          boolean createParent, int bufferSize, short replication, long blockSize,
256          Progressable progress) throws IOException {
257        if (exists(f) && !overwrite) {
258          throw new IOException("File already exists: "+f);
259        }
260        Path parent = f.getParent();
261        if (parent != null && !mkdirs(parent)) {
262          throw new IOException("Mkdirs failed to create " + parent.toString());
263        }
264        return new FSDataOutputStream(new BufferedOutputStream(
265            new LocalFSFileOutputStream(f, false), bufferSize), statistics);
266      }
267      
268      @Override
269      @Deprecated
270      public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
271          EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
272          Progressable progress) throws IOException {
273        if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) {
274          throw new IOException("File already exists: "+f);
275        }
276        return new FSDataOutputStream(new BufferedOutputStream(
277            new LocalFSFileOutputStream(f, false), bufferSize), statistics);
278      }
279    
280      /** {@inheritDoc} */
281      @Override
282      public FSDataOutputStream create(Path f, FsPermission permission,
283        boolean overwrite, int bufferSize, short replication, long blockSize,
284        Progressable progress) throws IOException {
285    
286        FSDataOutputStream out = create(f,
287            overwrite, bufferSize, replication, blockSize, progress);
288        setPermission(f, permission);
289        return out;
290      }
291    
292      /** {@inheritDoc} */
293      @Override
294      public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
295          boolean overwrite,
296          int bufferSize, short replication, long blockSize,
297          Progressable progress) throws IOException {
298        FSDataOutputStream out = create(f,
299            overwrite, false, bufferSize, replication, blockSize, progress);
300        setPermission(f, permission);
301        return out;
302      }
303    
304      public boolean rename(Path src, Path dst) throws IOException {
305        if (pathToFile(src).renameTo(pathToFile(dst))) {
306          return true;
307        }
308        return FileUtil.copy(this, src, this, dst, true, getConf());
309      }
310      
311      /**
312       * Delete the given path to a file or directory.
313       * @param p the path to delete
314       * @param recursive to delete sub-directories
315       * @return true if the file or directory and all its contents were deleted
316       * @throws IOException if p is non-empty and recursive is false 
317       */
318      public boolean delete(Path p, boolean recursive) throws IOException {
319        File f = pathToFile(p);
320        if (f.isFile()) {
321          return f.delete();
322        } else if (!recursive && f.isDirectory() && 
323            (FileUtil.listFiles(f).length != 0)) {
324          throw new IOException("Directory " + f.toString() + " is not empty");
325        }
326        return FileUtil.fullyDelete(f);
327      }
328     
329      public FileStatus[] listStatus(Path f) throws IOException {
330        File localf = pathToFile(f);
331        FileStatus[] results;
332    
333        if (!localf.exists()) {
334          throw new FileNotFoundException("File " + f + " does not exist");
335        }
336        if (localf.isFile()) {
337          return new FileStatus[] {
338            new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
339        }
340    
341        File[] names = localf.listFiles();
342        if (names == null) {
343          return null;
344        }
345        results = new FileStatus[names.length];
346        int j = 0;
347        for (int i = 0; i < names.length; i++) {
348          try {
349            results[j] = getFileStatus(new Path(names[i].getAbsolutePath()));
350            j++;
351          } catch (FileNotFoundException e) {
352            // ignore the files not found since the dir list may have have changed
353            // since the names[] list was generated.
354          }
355        }
356        if (j == names.length) {
357          return results;
358        }
359        return Arrays.copyOf(results, j);
360      }
361    
362      /**
363       * Creates the specified directory hierarchy. Does not
364       * treat existence as an error.
365       */
366      public boolean mkdirs(Path f) throws IOException {
367        if(f == null) {
368          throw new IllegalArgumentException("mkdirs path arg is null");
369        }
370        Path parent = f.getParent();
371        File p2f = pathToFile(f);
372        if(parent != null) {
373          File parent2f = pathToFile(parent);
374          if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
375            throw new FileAlreadyExistsException("Parent path is not a directory: " 
376                + parent);
377          }
378        }
379        return (parent == null || mkdirs(parent)) &&
380          (p2f.mkdir() || p2f.isDirectory());
381      }
382    
383      /** {@inheritDoc} */
384      @Override
385      public boolean mkdirs(Path f, FsPermission permission) throws IOException {
386        boolean b = mkdirs(f);
387        if(b) {
388          setPermission(f, permission);
389        }
390        return b;
391      }
392      
393    
394      @Override
395      protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
396        throws IOException {
397        boolean b = mkdirs(f);
398        setPermission(f, absolutePermission);
399        return b;
400      }
401      
402      
403      @Override
404      public Path getHomeDirectory() {
405        return this.makeQualified(new Path(System.getProperty("user.home")));
406      }
407    
408      /**
409       * Set the working directory to the given directory.
410       */
411      @Override
412      public void setWorkingDirectory(Path newDir) {
413        workingDir = makeAbsolute(newDir);
414        checkPath(workingDir);
415        
416      }
417      
418      @Override
419      public Path getWorkingDirectory() {
420        return workingDir;
421      }
422      
423      @Override
424      protected Path getInitialWorkingDirectory() {
425        return this.makeQualified(new Path(System.getProperty("user.dir")));
426      }
427    
428      /** {@inheritDoc} */
429      @Override
430      public FsStatus getStatus(Path p) throws IOException {
431        File partition = pathToFile(p == null ? new Path("/") : p);
432        //File provides getUsableSpace() and getFreeSpace()
433        //File provides no API to obtain used space, assume used = total - free
434        return new FsStatus(partition.getTotalSpace(), 
435          partition.getTotalSpace() - partition.getFreeSpace(),
436          partition.getFreeSpace());
437      }
438      
439      // In the case of the local filesystem, we can just rename the file.
440      public void moveFromLocalFile(Path src, Path dst) throws IOException {
441        rename(src, dst);
442      }
443      
444      // We can write output directly to the final location
445      public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
446        throws IOException {
447        return fsOutputFile;
448      }
449      
450      // It's in the right place - nothing to do.
451      public void completeLocalOutput(Path fsWorkingFile, Path tmpLocalFile)
452        throws IOException {
453      }
454      
455      public void close() throws IOException {
456        super.close();
457      }
458      
459      public String toString() {
460        return "LocalFS";
461      }
462      
463      public FileStatus getFileStatus(Path f) throws IOException {
464        File path = pathToFile(f);
465        if (path.exists()) {
466          return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this);
467        } else {
468          throw new FileNotFoundException("File " + f + " does not exist");
469        }
470      }
471    
472      static class RawLocalFileStatus extends FileStatus {
473        /* We can add extra fields here. It breaks at least CopyFiles.FilePair().
474         * We recognize if the information is already loaded by check if
475         * onwer.equals("").
476         */
477        private boolean isPermissionLoaded() {
478          return !super.getOwner().equals(""); 
479        }
480        
481        RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
482          super(f.length(), f.isDirectory(), 1, defaultBlockSize,
483                f.lastModified(), fs.makeQualified(new Path(f.getPath())));
484        }
485        
486        @Override
487        public FsPermission getPermission() {
488          if (!isPermissionLoaded()) {
489            loadPermissionInfo();
490          }
491          return super.getPermission();
492        }
493    
494        @Override
495        public String getOwner() {
496          if (!isPermissionLoaded()) {
497            loadPermissionInfo();
498          }
499          return super.getOwner();
500        }
501    
502        @Override
503        public String getGroup() {
504          if (!isPermissionLoaded()) {
505            loadPermissionInfo();
506          }
507          return super.getGroup();
508        }
509    
510        /// loads permissions, owner, and group from `ls -ld`
511        private void loadPermissionInfo() {
512          IOException e = null;
513          try {
514            StringTokenizer t = new StringTokenizer(
515                execCommand(new File(getPath().toUri()), 
516                            Shell.getGET_PERMISSION_COMMAND()));
517            //expected format
518            //-rw-------    1 username groupname ...
519            String permission = t.nextToken();
520            if (permission.length() > 10) { //files with ACLs might have a '+'
521              permission = permission.substring(0, 10);
522            }
523            setPermission(FsPermission.valueOf(permission));
524            t.nextToken();
525            setOwner(t.nextToken());
526            setGroup(t.nextToken());
527          } catch (Shell.ExitCodeException ioe) {
528            if (ioe.getExitCode() != 1) {
529              e = ioe;
530            } else {
531              setPermission(null);
532              setOwner(null);
533              setGroup(null);
534            }
535          } catch (IOException ioe) {
536            e = ioe;
537          } finally {
538            if (e != null) {
539              throw new RuntimeException("Error while running command to get " +
540                                         "file permissions : " + 
541                                         StringUtils.stringifyException(e));
542            }
543          }
544        }
545    
546        @Override
547        public void write(DataOutput out) throws IOException {
548          if (!isPermissionLoaded()) {
549            loadPermissionInfo();
550          }
551          super.write(out);
552        }
553      }
554    
555      /**
556       * Use the command chown to set owner.
557       */
558      @Override
559      public void setOwner(Path p, String username, String groupname)
560        throws IOException {
561        if (username == null && groupname == null) {
562          throw new IOException("username == null && groupname == null");
563        }
564    
565        if (username == null) {
566          execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); 
567        } else {
568          //OWNER[:[GROUP]]
569          String s = username + (groupname == null? "": ":" + groupname);
570          execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
571        }
572      }
573    
574      /**
575       * Use the command chmod to set permission.
576       */
577      @Override
578      public void setPermission(Path p, FsPermission permission)
579        throws IOException {
580        if (NativeIO.isAvailable()) {
581          NativeIO.chmod(pathToFile(p).getCanonicalPath(),
582                         permission.toShort());
583        } else {
584          execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
585              String.format("%05o", permission.toShort()));
586        }
587      }
588    
589      private static String execCommand(File f, String... cmd) throws IOException {
590        String[] args = new String[cmd.length + 1];
591        System.arraycopy(cmd, 0, args, 0, cmd.length);
592        args[cmd.length] = FileUtil.makeShellPath(f, true);
593        String output = Shell.execCommand(args);
594        return output;
595      }
596    
597    }