001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapreduce; 020 021 import java.io.FileNotFoundException; 022 import java.io.IOException; 023 import java.net.InetSocketAddress; 024 import java.security.PrivilegedExceptionAction; 025 import java.util.ArrayList; 026 import java.util.List; 027 import java.util.ServiceLoader; 028 029 import org.apache.commons.logging.Log; 030 import org.apache.commons.logging.LogFactory; 031 import org.apache.hadoop.classification.InterfaceAudience; 032 import org.apache.hadoop.classification.InterfaceStability; 033 import org.apache.hadoop.conf.Configuration; 034 import org.apache.hadoop.fs.FileSystem; 035 import org.apache.hadoop.fs.Path; 036 import org.apache.hadoop.io.Text; 037 import org.apache.hadoop.ipc.RemoteException; 038 import org.apache.hadoop.mapred.JobConf; 039 import org.apache.hadoop.mapreduce.protocol.ClientProtocol; 040 import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider; 041 import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; 042 import org.apache.hadoop.mapreduce.util.ConfigUtil; 043 import org.apache.hadoop.mapreduce.v2.LogParams; 044 import org.apache.hadoop.security.AccessControlException; 045 import org.apache.hadoop.security.UserGroupInformation; 046 import org.apache.hadoop.security.token.SecretManager.InvalidToken; 047 import org.apache.hadoop.security.token.Token; 048 049 /** 050 * Provides a way to access information about the map/reduce cluster. 051 */ 052 @InterfaceAudience.Public 053 @InterfaceStability.Evolving 054 public class Cluster { 055 056 @InterfaceStability.Evolving 057 public static enum JobTrackerStatus {INITIALIZING, RUNNING}; 058 059 private ClientProtocolProvider clientProtocolProvider; 060 private ClientProtocol client; 061 private UserGroupInformation ugi; 062 private Configuration conf; 063 private FileSystem fs = null; 064 private Path sysDir = null; 065 private Path stagingAreaDir = null; 066 private Path jobHistoryDir = null; 067 private static final Log LOG = LogFactory.getLog(Cluster.class); 068 069 private static ServiceLoader<ClientProtocolProvider> frameworkLoader = 070 ServiceLoader.load(ClientProtocolProvider.class); 071 072 static { 073 ConfigUtil.loadResources(); 074 } 075 076 public Cluster(Configuration conf) throws IOException { 077 this(null, conf); 078 } 079 080 public Cluster(InetSocketAddress jobTrackAddr, Configuration conf) 081 throws IOException { 082 this.conf = conf; 083 this.ugi = UserGroupInformation.getCurrentUser(); 084 initialize(jobTrackAddr, conf); 085 } 086 087 private void initialize(InetSocketAddress jobTrackAddr, Configuration conf) 088 throws IOException { 089 090 synchronized (frameworkLoader) { 091 for (ClientProtocolProvider provider : frameworkLoader) { 092 LOG.debug("Trying ClientProtocolProvider : " 093 + provider.getClass().getName()); 094 ClientProtocol clientProtocol = null; 095 try { 096 if (jobTrackAddr == null) { 097 clientProtocol = provider.create(conf); 098 } else { 099 clientProtocol = provider.create(jobTrackAddr, conf); 100 } 101 102 if (clientProtocol != null) { 103 clientProtocolProvider = provider; 104 client = clientProtocol; 105 LOG.debug("Picked " + provider.getClass().getName() 106 + " as the ClientProtocolProvider"); 107 break; 108 } 109 else { 110 LOG.debug("Cannot pick " + provider.getClass().getName() 111 + " as the ClientProtocolProvider - returned null protocol"); 112 } 113 } 114 catch (Exception e) { 115 LOG.info("Failed to use " + provider.getClass().getName() 116 + " due to error: " + e.getMessage()); 117 } 118 } 119 } 120 121 if (null == clientProtocolProvider || null == client) { 122 throw new IOException( 123 "Cannot initialize Cluster. Please check your configuration for " 124 + MRConfig.FRAMEWORK_NAME 125 + " and the correspond server addresses."); 126 } 127 } 128 129 ClientProtocol getClient() { 130 return client; 131 } 132 133 Configuration getConf() { 134 return conf; 135 } 136 137 /** 138 * Close the <code>Cluster</code>. 139 */ 140 public synchronized void close() throws IOException { 141 clientProtocolProvider.close(client); 142 } 143 144 private Job[] getJobs(JobStatus[] stats) throws IOException { 145 List<Job> jobs = new ArrayList<Job>(); 146 for (JobStatus stat : stats) { 147 jobs.add(Job.getInstance(this, stat, new JobConf(stat.getJobFile()))); 148 } 149 return jobs.toArray(new Job[0]); 150 } 151 152 /** 153 * Get the file system where job-specific files are stored 154 * 155 * @return object of FileSystem 156 * @throws IOException 157 * @throws InterruptedException 158 */ 159 public synchronized FileSystem getFileSystem() 160 throws IOException, InterruptedException { 161 if (this.fs == null) { 162 try { 163 this.fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { 164 public FileSystem run() throws IOException, InterruptedException { 165 final Path sysDir = new Path(client.getSystemDir()); 166 return sysDir.getFileSystem(getConf()); 167 } 168 }); 169 } catch (InterruptedException e) { 170 throw new RuntimeException(e); 171 } 172 } 173 return fs; 174 } 175 176 /** 177 * Get job corresponding to jobid. 178 * 179 * @param jobId 180 * @return object of {@link Job} 181 * @throws IOException 182 * @throws InterruptedException 183 */ 184 public Job getJob(JobID jobId) throws IOException, InterruptedException { 185 JobStatus status = client.getJobStatus(jobId); 186 if (status != null) { 187 JobConf conf; 188 try { 189 conf = new JobConf(status.getJobFile()); 190 } catch (RuntimeException ex) { 191 // If job file doesn't exist it means we can't find the job 192 if (ex.getCause() instanceof FileNotFoundException) { 193 return null; 194 } else { 195 throw ex; 196 } 197 } 198 return Job.getInstance(this, status, conf); 199 } 200 return null; 201 } 202 203 /** 204 * Get all the queues in cluster. 205 * 206 * @return array of {@link QueueInfo} 207 * @throws IOException 208 * @throws InterruptedException 209 */ 210 public QueueInfo[] getQueues() throws IOException, InterruptedException { 211 return client.getQueues(); 212 } 213 214 /** 215 * Get queue information for the specified name. 216 * 217 * @param name queuename 218 * @return object of {@link QueueInfo} 219 * @throws IOException 220 * @throws InterruptedException 221 */ 222 public QueueInfo getQueue(String name) 223 throws IOException, InterruptedException { 224 return client.getQueue(name); 225 } 226 227 /** 228 * Get log parameters for the specified jobID or taskAttemptID 229 * @param jobID the job id. 230 * @param taskAttemptID the task attempt id. Optional. 231 * @return the LogParams 232 * @throws IOException 233 * @throws InterruptedException 234 */ 235 public LogParams getLogParams(JobID jobID, TaskAttemptID taskAttemptID) 236 throws IOException, InterruptedException { 237 return client.getLogFileParams(jobID, taskAttemptID); 238 } 239 240 /** 241 * Get current cluster status. 242 * 243 * @return object of {@link ClusterMetrics} 244 * @throws IOException 245 * @throws InterruptedException 246 */ 247 public ClusterMetrics getClusterStatus() throws IOException, InterruptedException { 248 return client.getClusterMetrics(); 249 } 250 251 /** 252 * Get all active trackers in the cluster. 253 * 254 * @return array of {@link TaskTrackerInfo} 255 * @throws IOException 256 * @throws InterruptedException 257 */ 258 public TaskTrackerInfo[] getActiveTaskTrackers() 259 throws IOException, InterruptedException { 260 return client.getActiveTrackers(); 261 } 262 263 /** 264 * Get blacklisted trackers. 265 * 266 * @return array of {@link TaskTrackerInfo} 267 * @throws IOException 268 * @throws InterruptedException 269 */ 270 public TaskTrackerInfo[] getBlackListedTaskTrackers() 271 throws IOException, InterruptedException { 272 return client.getBlacklistedTrackers(); 273 } 274 275 /** 276 * Get all the jobs in cluster. 277 * 278 * @return array of {@link Job} 279 * @throws IOException 280 * @throws InterruptedException 281 * @deprecated Use {@link #getAllJobStatuses()} instead. 282 */ 283 @Deprecated 284 public Job[] getAllJobs() throws IOException, InterruptedException { 285 return getJobs(client.getAllJobs()); 286 } 287 288 /** 289 * Get job status for all jobs in the cluster. 290 * @return job status for all jobs in cluster 291 * @throws IOException 292 * @throws InterruptedException 293 */ 294 public JobStatus[] getAllJobStatuses() throws IOException, InterruptedException { 295 return client.getAllJobs(); 296 } 297 298 /** 299 * Grab the jobtracker system directory path where 300 * job-specific files will be placed. 301 * 302 * @return the system directory where job-specific files are to be placed. 303 */ 304 public Path getSystemDir() throws IOException, InterruptedException { 305 if (sysDir == null) { 306 sysDir = new Path(client.getSystemDir()); 307 } 308 return sysDir; 309 } 310 311 /** 312 * Grab the jobtracker's view of the staging directory path where 313 * job-specific files will be placed. 314 * 315 * @return the staging directory where job-specific files are to be placed. 316 */ 317 public Path getStagingAreaDir() throws IOException, InterruptedException { 318 if (stagingAreaDir == null) { 319 stagingAreaDir = new Path(client.getStagingAreaDir()); 320 } 321 return stagingAreaDir; 322 } 323 324 /** 325 * Get the job history file path for a given job id. The job history file at 326 * this path may or may not be existing depending on the job completion state. 327 * The file is present only for the completed jobs. 328 * @param jobId the JobID of the job submitted by the current user. 329 * @return the file path of the job history file 330 * @throws IOException 331 * @throws InterruptedException 332 */ 333 public String getJobHistoryUrl(JobID jobId) throws IOException, 334 InterruptedException { 335 if (jobHistoryDir == null) { 336 jobHistoryDir = new Path(client.getJobHistoryDir()); 337 } 338 return new Path(jobHistoryDir, jobId.toString() + "_" 339 + ugi.getShortUserName()).toString(); 340 } 341 342 /** 343 * Gets the Queue ACLs for current user 344 * @return array of QueueAclsInfo object for current user. 345 * @throws IOException 346 */ 347 public QueueAclsInfo[] getQueueAclsForCurrentUser() 348 throws IOException, InterruptedException { 349 return client.getQueueAclsForCurrentUser(); 350 } 351 352 /** 353 * Gets the root level queues. 354 * @return array of JobQueueInfo object. 355 * @throws IOException 356 */ 357 public QueueInfo[] getRootQueues() throws IOException, InterruptedException { 358 return client.getRootQueues(); 359 } 360 361 /** 362 * Returns immediate children of queueName. 363 * @param queueName 364 * @return array of JobQueueInfo which are children of queueName 365 * @throws IOException 366 */ 367 public QueueInfo[] getChildQueues(String queueName) 368 throws IOException, InterruptedException { 369 return client.getChildQueues(queueName); 370 } 371 372 /** 373 * Get the JobTracker's status. 374 * 375 * @return {@link JobTrackerStatus} of the JobTracker 376 * @throws IOException 377 * @throws InterruptedException 378 */ 379 public JobTrackerStatus getJobTrackerStatus() throws IOException, 380 InterruptedException { 381 return client.getJobTrackerStatus(); 382 } 383 384 /** 385 * Get the tasktracker expiry interval for the cluster 386 * @return the expiry interval in msec 387 */ 388 public long getTaskTrackerExpiryInterval() throws IOException, 389 InterruptedException { 390 return client.getTaskTrackerExpiryInterval(); 391 } 392 393 /** 394 * Get a delegation token for the user from the JobTracker. 395 * @param renewer the user who can renew the token 396 * @return the new token 397 * @throws IOException 398 */ 399 public Token<DelegationTokenIdentifier> 400 getDelegationToken(Text renewer) throws IOException, InterruptedException{ 401 // client has already set the service 402 return client.getDelegationToken(renewer); 403 } 404 405 /** 406 * Renew a delegation token 407 * @param token the token to renew 408 * @return the new expiration time 409 * @throws InvalidToken 410 * @throws IOException 411 * @deprecated Use {@link Token#renew} instead 412 */ 413 public long renewDelegationToken(Token<DelegationTokenIdentifier> token 414 ) throws InvalidToken, IOException, 415 InterruptedException { 416 try { 417 return client.renewDelegationToken(token); 418 } catch (RemoteException re) { 419 throw re.unwrapRemoteException(InvalidToken.class, 420 AccessControlException.class); 421 } 422 } 423 424 /** 425 * Cancel a delegation token from the JobTracker 426 * @param token the token to cancel 427 * @throws IOException 428 * @deprecated Use {@link Token#cancel} instead 429 */ 430 public void cancelDelegationToken(Token<DelegationTokenIdentifier> token 431 ) throws IOException, 432 InterruptedException { 433 try { 434 client.cancelDelegationToken(token); 435 } catch (RemoteException re) { 436 throw re.unwrapRemoteException(InvalidToken.class, 437 AccessControlException.class); 438 } 439 } 440 441 }