001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.hadoop.fs.s3;
019    
020    import java.io.IOException;
021    import java.io.InputStream;
022    import java.io.UnsupportedEncodingException;
023    import java.net.URI;
024    import java.net.URLDecoder;
025    import java.net.URLEncoder;
026    import java.util.Set;
027    import java.util.TreeSet;
028    
029    import org.apache.hadoop.classification.InterfaceAudience;
030    import org.apache.hadoop.classification.InterfaceStability;
031    import org.apache.hadoop.conf.Configured;
032    import org.apache.hadoop.fs.Path;
033    import org.apache.hadoop.util.Tool;
034    import org.apache.hadoop.util.ToolRunner;
035    import com.cloudera.org.jets3t.service.S3Service;
036    import com.cloudera.org.jets3t.service.S3ServiceException;
037    import com.cloudera.org.jets3t.service.ServiceException;
038    import com.cloudera.org.jets3t.service.impl.rest.httpclient.RestS3Service;
039    import com.cloudera.org.jets3t.service.model.S3Bucket;
040    import com.cloudera.org.jets3t.service.model.S3Object;
041    import com.cloudera.org.jets3t.service.security.AWSCredentials;
042    
043    /**
044     * <p>
045     * This class is a tool for migrating data from an older to a newer version
046     * of an S3 filesystem.
047     * </p>
048     * <p>
049     * All files in the filesystem are migrated by re-writing the block metadata
050     * - no datafiles are touched.
051     * </p>
052     */
053    @InterfaceAudience.Public
054    @InterfaceStability.Unstable
055    public class MigrationTool extends Configured implements Tool {
056      
057      private S3Service s3Service;
058      private S3Bucket bucket;
059      
060      public static void main(String[] args) throws Exception {
061        int res = ToolRunner.run(new MigrationTool(), args);
062        System.exit(res);
063      }
064      
065      public int run(String[] args) throws Exception {
066        
067        if (args.length == 0) {
068          System.err.println("Usage: MigrationTool <S3 file system URI>");
069          System.err.println("\t<S3 file system URI>\tfilesystem to migrate");
070          ToolRunner.printGenericCommandUsage(System.err);
071          return -1;
072        }
073        
074        URI uri = URI.create(args[0]);
075        
076        initialize(uri);
077        
078        FileSystemStore newStore = new Jets3tFileSystemStore();
079        newStore.initialize(uri, getConf());
080        
081        if (get("%2F") != null) { 
082          System.err.println("Current version number is [unversioned].");
083          System.err.println("Target version number is " +
084              newStore.getVersion() + ".");
085          Store oldStore = new UnversionedStore();
086          migrate(oldStore, newStore);
087          return 0;
088        } else {
089          S3Object root = get("/");
090          if (root != null) {
091            String version = (String) root.getMetadata("fs-version");
092            if (version == null) {
093              System.err.println("Can't detect version - exiting.");
094            } else {
095              String newVersion = newStore.getVersion();
096              System.err.println("Current version number is " + version + ".");
097              System.err.println("Target version number is " + newVersion + ".");
098              if (version.equals(newStore.getVersion())) {
099                System.err.println("No migration required.");
100                return 0;
101              }
102              // use version number to create Store
103              //Store oldStore = ... 
104              //migrate(oldStore, newStore);
105              System.err.println("Not currently implemented.");
106              return 0;
107            }
108          }
109          System.err.println("Can't detect version - exiting.");
110          return 0;
111        }
112        
113      }
114      
115      public void initialize(URI uri) throws IOException {
116        
117        
118        
119        try {
120          String accessKey = null;
121          String secretAccessKey = null;
122          String userInfo = uri.getUserInfo();
123          if (userInfo != null) {
124            int index = userInfo.indexOf(':');
125            if (index != -1) {
126              accessKey = userInfo.substring(0, index);
127              secretAccessKey = userInfo.substring(index + 1);
128            } else {
129              accessKey = userInfo;
130            }
131          }
132          if (accessKey == null) {
133            accessKey = getConf().get("fs.s3.awsAccessKeyId");
134          }
135          if (secretAccessKey == null) {
136            secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey");
137          }
138          if (accessKey == null && secretAccessKey == null) {
139            throw new IllegalArgumentException("AWS " +
140                                               "Access Key ID and Secret Access Key " +
141                                               "must be specified as the username " +
142                                               "or password (respectively) of a s3 URL, " +
143                                               "or by setting the " +
144                                               "fs.s3.awsAccessKeyId or " +                         
145                                               "fs.s3.awsSecretAccessKey properties (respectively).");
146          } else if (accessKey == null) {
147            throw new IllegalArgumentException("AWS " +
148                                               "Access Key ID must be specified " +
149                                               "as the username of a s3 URL, or by setting the " +
150                                               "fs.s3.awsAccessKeyId property.");
151          } else if (secretAccessKey == null) {
152            throw new IllegalArgumentException("AWS " +
153                                               "Secret Access Key must be specified " +
154                                               "as the password of a s3 URL, or by setting the " +
155                                               "fs.s3.awsSecretAccessKey property.");         
156          }
157          AWSCredentials awsCredentials =
158            new AWSCredentials(accessKey, secretAccessKey);
159          this.s3Service = new RestS3Service(awsCredentials);
160        } catch (S3ServiceException e) {
161          if (e.getCause() instanceof IOException) {
162            throw (IOException) e.getCause();
163          }
164          throw new S3Exception(e);
165        }
166        bucket = new S3Bucket(uri.getHost());
167      }
168      
169      private void migrate(Store oldStore, FileSystemStore newStore)
170          throws IOException {
171        for (Path path : oldStore.listAllPaths()) {
172          INode inode = oldStore.retrieveINode(path);
173          oldStore.deleteINode(path);
174          newStore.storeINode(path, inode);
175        }
176      }
177      
178      private S3Object get(String key) {
179        try {
180          return s3Service.getObject(bucket.getName(), key);
181        } catch (S3ServiceException e) {
182          if ("NoSuchKey".equals(e.getS3ErrorCode())) {
183            return null;
184          }
185        }
186        return null;
187      }
188      
189      interface Store {
190    
191        Set<Path> listAllPaths() throws IOException;
192        INode retrieveINode(Path path) throws IOException;
193        void deleteINode(Path path) throws IOException;
194        
195      }
196      
197      class UnversionedStore implements Store {
198    
199        public Set<Path> listAllPaths() throws IOException {
200          try {
201            String prefix = urlEncode(Path.SEPARATOR);
202            S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null);
203            Set<Path> prefixes = new TreeSet<Path>();
204            for (int i = 0; i < objects.length; i++) {
205              prefixes.add(keyToPath(objects[i].getKey()));
206            }
207            return prefixes;
208          } catch (S3ServiceException e) {
209            if (e.getCause() instanceof IOException) {
210              throw (IOException) e.getCause();
211            }
212            throw new S3Exception(e);
213          }   
214        }
215    
216        public void deleteINode(Path path) throws IOException {
217          delete(pathToKey(path));
218        }
219        
220        private void delete(String key) throws IOException {
221          try {
222            s3Service.deleteObject(bucket, key);
223          } catch (S3ServiceException e) {
224            if (e.getCause() instanceof IOException) {
225              throw (IOException) e.getCause();
226            }
227            throw new S3Exception(e);
228          }
229        }
230        
231        public INode retrieveINode(Path path) throws IOException {
232          return INode.deserialize(get(pathToKey(path)));
233        }
234    
235        private InputStream get(String key) throws IOException {
236          try {
237            S3Object object = s3Service.getObject(bucket.getName(), key);
238            return object.getDataInputStream();
239          } catch (S3ServiceException e) {
240            if ("NoSuchKey".equals(e.getS3ErrorCode())) {
241              return null;
242            }
243            if (e.getCause() instanceof IOException) {
244              throw (IOException) e.getCause();
245            }
246            throw new S3Exception(e);
247          } catch (ServiceException e) {
248            return null;
249          }
250        }
251        
252        private String pathToKey(Path path) {
253          if (!path.isAbsolute()) {
254            throw new IllegalArgumentException("Path must be absolute: " + path);
255          }
256          return urlEncode(path.toUri().getPath());
257        }
258        
259        private Path keyToPath(String key) {
260          return new Path(urlDecode(key));
261        }
262    
263        private String urlEncode(String s) {
264          try {
265            return URLEncoder.encode(s, "UTF-8");
266          } catch (UnsupportedEncodingException e) {
267            // Should never happen since every implementation of the Java Platform
268            // is required to support UTF-8.
269            // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html
270            throw new IllegalStateException(e);
271          }
272        }
273        
274        private String urlDecode(String s) {
275          try {
276            return URLDecoder.decode(s, "UTF-8");
277          } catch (UnsupportedEncodingException e) {
278            // Should never happen since every implementation of the Java Platform
279            // is required to support UTF-8.
280            // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html
281            throw new IllegalStateException(e);
282          }
283        }
284        
285      }
286      
287    }