001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 package org.apache.hadoop.fs.s3; 019 020 import java.io.IOException; 021 import java.io.InputStream; 022 import java.io.UnsupportedEncodingException; 023 import java.net.URI; 024 import java.net.URLDecoder; 025 import java.net.URLEncoder; 026 import java.util.Set; 027 import java.util.TreeSet; 028 029 import org.apache.hadoop.classification.InterfaceAudience; 030 import org.apache.hadoop.classification.InterfaceStability; 031 import org.apache.hadoop.conf.Configured; 032 import org.apache.hadoop.fs.Path; 033 import org.apache.hadoop.util.Tool; 034 import org.apache.hadoop.util.ToolRunner; 035 import com.cloudera.org.jets3t.service.S3Service; 036 import com.cloudera.org.jets3t.service.S3ServiceException; 037 import com.cloudera.org.jets3t.service.ServiceException; 038 import com.cloudera.org.jets3t.service.impl.rest.httpclient.RestS3Service; 039 import com.cloudera.org.jets3t.service.model.S3Bucket; 040 import com.cloudera.org.jets3t.service.model.S3Object; 041 import com.cloudera.org.jets3t.service.security.AWSCredentials; 042 043 /** 044 * <p> 045 * This class is a tool for migrating data from an older to a newer version 046 * of an S3 filesystem. 047 * </p> 048 * <p> 049 * All files in the filesystem are migrated by re-writing the block metadata 050 * - no datafiles are touched. 051 * </p> 052 */ 053 @InterfaceAudience.Public 054 @InterfaceStability.Unstable 055 public class MigrationTool extends Configured implements Tool { 056 057 private S3Service s3Service; 058 private S3Bucket bucket; 059 060 public static void main(String[] args) throws Exception { 061 int res = ToolRunner.run(new MigrationTool(), args); 062 System.exit(res); 063 } 064 065 public int run(String[] args) throws Exception { 066 067 if (args.length == 0) { 068 System.err.println("Usage: MigrationTool <S3 file system URI>"); 069 System.err.println("\t<S3 file system URI>\tfilesystem to migrate"); 070 ToolRunner.printGenericCommandUsage(System.err); 071 return -1; 072 } 073 074 URI uri = URI.create(args[0]); 075 076 initialize(uri); 077 078 FileSystemStore newStore = new Jets3tFileSystemStore(); 079 newStore.initialize(uri, getConf()); 080 081 if (get("%2F") != null) { 082 System.err.println("Current version number is [unversioned]."); 083 System.err.println("Target version number is " + 084 newStore.getVersion() + "."); 085 Store oldStore = new UnversionedStore(); 086 migrate(oldStore, newStore); 087 return 0; 088 } else { 089 S3Object root = get("/"); 090 if (root != null) { 091 String version = (String) root.getMetadata("fs-version"); 092 if (version == null) { 093 System.err.println("Can't detect version - exiting."); 094 } else { 095 String newVersion = newStore.getVersion(); 096 System.err.println("Current version number is " + version + "."); 097 System.err.println("Target version number is " + newVersion + "."); 098 if (version.equals(newStore.getVersion())) { 099 System.err.println("No migration required."); 100 return 0; 101 } 102 // use version number to create Store 103 //Store oldStore = ... 104 //migrate(oldStore, newStore); 105 System.err.println("Not currently implemented."); 106 return 0; 107 } 108 } 109 System.err.println("Can't detect version - exiting."); 110 return 0; 111 } 112 113 } 114 115 public void initialize(URI uri) throws IOException { 116 117 118 119 try { 120 String accessKey = null; 121 String secretAccessKey = null; 122 String userInfo = uri.getUserInfo(); 123 if (userInfo != null) { 124 int index = userInfo.indexOf(':'); 125 if (index != -1) { 126 accessKey = userInfo.substring(0, index); 127 secretAccessKey = userInfo.substring(index + 1); 128 } else { 129 accessKey = userInfo; 130 } 131 } 132 if (accessKey == null) { 133 accessKey = getConf().get("fs.s3.awsAccessKeyId"); 134 } 135 if (secretAccessKey == null) { 136 secretAccessKey = getConf().get("fs.s3.awsSecretAccessKey"); 137 } 138 if (accessKey == null && secretAccessKey == null) { 139 throw new IllegalArgumentException("AWS " + 140 "Access Key ID and Secret Access Key " + 141 "must be specified as the username " + 142 "or password (respectively) of a s3 URL, " + 143 "or by setting the " + 144 "fs.s3.awsAccessKeyId or " + 145 "fs.s3.awsSecretAccessKey properties (respectively)."); 146 } else if (accessKey == null) { 147 throw new IllegalArgumentException("AWS " + 148 "Access Key ID must be specified " + 149 "as the username of a s3 URL, or by setting the " + 150 "fs.s3.awsAccessKeyId property."); 151 } else if (secretAccessKey == null) { 152 throw new IllegalArgumentException("AWS " + 153 "Secret Access Key must be specified " + 154 "as the password of a s3 URL, or by setting the " + 155 "fs.s3.awsSecretAccessKey property."); 156 } 157 AWSCredentials awsCredentials = 158 new AWSCredentials(accessKey, secretAccessKey); 159 this.s3Service = new RestS3Service(awsCredentials); 160 } catch (S3ServiceException e) { 161 if (e.getCause() instanceof IOException) { 162 throw (IOException) e.getCause(); 163 } 164 throw new S3Exception(e); 165 } 166 bucket = new S3Bucket(uri.getHost()); 167 } 168 169 private void migrate(Store oldStore, FileSystemStore newStore) 170 throws IOException { 171 for (Path path : oldStore.listAllPaths()) { 172 INode inode = oldStore.retrieveINode(path); 173 oldStore.deleteINode(path); 174 newStore.storeINode(path, inode); 175 } 176 } 177 178 private S3Object get(String key) { 179 try { 180 return s3Service.getObject(bucket.getName(), key); 181 } catch (S3ServiceException e) { 182 if ("NoSuchKey".equals(e.getS3ErrorCode())) { 183 return null; 184 } 185 } 186 return null; 187 } 188 189 interface Store { 190 191 Set<Path> listAllPaths() throws IOException; 192 INode retrieveINode(Path path) throws IOException; 193 void deleteINode(Path path) throws IOException; 194 195 } 196 197 class UnversionedStore implements Store { 198 199 public Set<Path> listAllPaths() throws IOException { 200 try { 201 String prefix = urlEncode(Path.SEPARATOR); 202 S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null); 203 Set<Path> prefixes = new TreeSet<Path>(); 204 for (int i = 0; i < objects.length; i++) { 205 prefixes.add(keyToPath(objects[i].getKey())); 206 } 207 return prefixes; 208 } catch (S3ServiceException e) { 209 if (e.getCause() instanceof IOException) { 210 throw (IOException) e.getCause(); 211 } 212 throw new S3Exception(e); 213 } 214 } 215 216 public void deleteINode(Path path) throws IOException { 217 delete(pathToKey(path)); 218 } 219 220 private void delete(String key) throws IOException { 221 try { 222 s3Service.deleteObject(bucket, key); 223 } catch (S3ServiceException e) { 224 if (e.getCause() instanceof IOException) { 225 throw (IOException) e.getCause(); 226 } 227 throw new S3Exception(e); 228 } 229 } 230 231 public INode retrieveINode(Path path) throws IOException { 232 return INode.deserialize(get(pathToKey(path))); 233 } 234 235 private InputStream get(String key) throws IOException { 236 try { 237 S3Object object = s3Service.getObject(bucket.getName(), key); 238 return object.getDataInputStream(); 239 } catch (S3ServiceException e) { 240 if ("NoSuchKey".equals(e.getS3ErrorCode())) { 241 return null; 242 } 243 if (e.getCause() instanceof IOException) { 244 throw (IOException) e.getCause(); 245 } 246 throw new S3Exception(e); 247 } catch (ServiceException e) { 248 return null; 249 } 250 } 251 252 private String pathToKey(Path path) { 253 if (!path.isAbsolute()) { 254 throw new IllegalArgumentException("Path must be absolute: " + path); 255 } 256 return urlEncode(path.toUri().getPath()); 257 } 258 259 private Path keyToPath(String key) { 260 return new Path(urlDecode(key)); 261 } 262 263 private String urlEncode(String s) { 264 try { 265 return URLEncoder.encode(s, "UTF-8"); 266 } catch (UnsupportedEncodingException e) { 267 // Should never happen since every implementation of the Java Platform 268 // is required to support UTF-8. 269 // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html 270 throw new IllegalStateException(e); 271 } 272 } 273 274 private String urlDecode(String s) { 275 try { 276 return URLDecoder.decode(s, "UTF-8"); 277 } catch (UnsupportedEncodingException e) { 278 // Should never happen since every implementation of the Java Platform 279 // is required to support UTF-8. 280 // See http://java.sun.com/j2se/1.5.0/docs/api/java/nio/charset/Charset.html 281 throw new IllegalStateException(e); 282 } 283 } 284 285 } 286 287 }