001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapreduce;
020    
021    import java.io.FileNotFoundException;
022    import java.io.IOException;
023    import java.net.InetSocketAddress;
024    import java.security.PrivilegedExceptionAction;
025    import java.util.ArrayList;
026    import java.util.List;
027    import java.util.ServiceLoader;
028    
029    import org.apache.commons.logging.Log;
030    import org.apache.commons.logging.LogFactory;
031    import org.apache.hadoop.classification.InterfaceAudience;
032    import org.apache.hadoop.classification.InterfaceStability;
033    import org.apache.hadoop.conf.Configuration;
034    import org.apache.hadoop.fs.FileSystem;
035    import org.apache.hadoop.fs.Path;
036    import org.apache.hadoop.io.Text;
037    import org.apache.hadoop.ipc.RemoteException;
038    import org.apache.hadoop.mapred.JobConf;
039    import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
040    import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
041    import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
042    import org.apache.hadoop.mapreduce.util.ConfigUtil;
043    import org.apache.hadoop.mapreduce.v2.LogParams;
044    import org.apache.hadoop.security.AccessControlException;
045    import org.apache.hadoop.security.UserGroupInformation;
046    import org.apache.hadoop.security.token.SecretManager.InvalidToken;
047    import org.apache.hadoop.security.token.Token;
048    
049    /**
050     * Provides a way to access information about the map/reduce cluster.
051     */
052    @InterfaceAudience.Public
053    @InterfaceStability.Evolving
054    public class Cluster {
055      
056      @InterfaceStability.Evolving
057      public static enum JobTrackerStatus {INITIALIZING, RUNNING};
058      
059      private ClientProtocolProvider clientProtocolProvider;
060      private ClientProtocol client;
061      private UserGroupInformation ugi;
062      private Configuration conf;
063      private FileSystem fs = null;
064      private Path sysDir = null;
065      private Path stagingAreaDir = null;
066      private Path jobHistoryDir = null;
067      private static final Log LOG = LogFactory.getLog(Cluster.class);
068    
069      private static ServiceLoader<ClientProtocolProvider> frameworkLoader =
070          ServiceLoader.load(ClientProtocolProvider.class);
071      
072      static {
073        ConfigUtil.loadResources();
074      }
075      
076      public Cluster(Configuration conf) throws IOException {
077        this(null, conf);
078      }
079    
080      public Cluster(InetSocketAddress jobTrackAddr, Configuration conf) 
081          throws IOException {
082        this.conf = conf;
083        this.ugi = UserGroupInformation.getCurrentUser();
084        initialize(jobTrackAddr, conf);
085      }
086      
087      private void initialize(InetSocketAddress jobTrackAddr, Configuration conf)
088          throws IOException {
089    
090        synchronized (frameworkLoader) {
091          for (ClientProtocolProvider provider : frameworkLoader) {
092            LOG.debug("Trying ClientProtocolProvider : "
093                + provider.getClass().getName());
094            ClientProtocol clientProtocol = null; 
095            try {
096              if (jobTrackAddr == null) {
097                clientProtocol = provider.create(conf);
098              } else {
099                clientProtocol = provider.create(jobTrackAddr, conf);
100              }
101    
102              if (clientProtocol != null) {
103                clientProtocolProvider = provider;
104                client = clientProtocol;
105                LOG.debug("Picked " + provider.getClass().getName()
106                    + " as the ClientProtocolProvider");
107                break;
108              }
109              else {
110                LOG.debug("Cannot pick " + provider.getClass().getName()
111                    + " as the ClientProtocolProvider - returned null protocol");
112              }
113            } 
114            catch (Exception e) {
115              LOG.info("Failed to use " + provider.getClass().getName()
116                  + " due to error: " + e.getMessage());
117            }
118          }
119        }
120    
121        if (null == clientProtocolProvider || null == client) {
122          throw new IOException(
123              "Cannot initialize Cluster. Please check your configuration for "
124                  + MRConfig.FRAMEWORK_NAME
125                  + " and the correspond server addresses.");
126        }
127      }
128    
129      ClientProtocol getClient() {
130        return client;
131      }
132      
133      Configuration getConf() {
134        return conf;
135      }
136      
137      /**
138       * Close the <code>Cluster</code>.
139       */
140      public synchronized void close() throws IOException {
141        clientProtocolProvider.close(client);
142      }
143    
144      private Job[] getJobs(JobStatus[] stats) throws IOException {
145        List<Job> jobs = new ArrayList<Job>();
146        for (JobStatus stat : stats) {
147          jobs.add(Job.getInstance(this, stat, new JobConf(stat.getJobFile())));
148        }
149        return jobs.toArray(new Job[0]);
150      }
151    
152      /**
153       * Get the file system where job-specific files are stored
154       * 
155       * @return object of FileSystem
156       * @throws IOException
157       * @throws InterruptedException
158       */
159      public synchronized FileSystem getFileSystem() 
160          throws IOException, InterruptedException {
161        if (this.fs == null) {
162          try {
163            this.fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
164              public FileSystem run() throws IOException, InterruptedException {
165                final Path sysDir = new Path(client.getSystemDir());
166                return sysDir.getFileSystem(getConf());
167              }
168            });
169          } catch (InterruptedException e) {
170            throw new RuntimeException(e);
171          }
172        }
173        return fs;
174      }
175    
176      /**
177       * Get job corresponding to jobid.
178       * 
179       * @param jobId
180       * @return object of {@link Job}
181       * @throws IOException
182       * @throws InterruptedException
183       */
184      public Job getJob(JobID jobId) throws IOException, InterruptedException {
185        JobStatus status = client.getJobStatus(jobId);
186        if (status != null) {
187          JobConf conf;
188          try {
189            conf = new JobConf(status.getJobFile());
190          } catch (RuntimeException ex) {
191            // If job file doesn't exist it means we can't find the job
192            if (ex.getCause() instanceof FileNotFoundException) {
193              return null;
194            } else {
195              throw ex;
196            }
197          }
198          return Job.getInstance(this, status, conf);
199        }
200        return null;
201      }
202      
203      /**
204       * Get all the queues in cluster.
205       * 
206       * @return array of {@link QueueInfo}
207       * @throws IOException
208       * @throws InterruptedException
209       */
210      public QueueInfo[] getQueues() throws IOException, InterruptedException {
211        return client.getQueues();
212      }
213      
214      /**
215       * Get queue information for the specified name.
216       * 
217       * @param name queuename
218       * @return object of {@link QueueInfo}
219       * @throws IOException
220       * @throws InterruptedException
221       */
222      public QueueInfo getQueue(String name) 
223          throws IOException, InterruptedException {
224        return client.getQueue(name);
225      }
226    
227      /**
228       * Get log parameters for the specified jobID or taskAttemptID
229       * @param jobID the job id.
230       * @param taskAttemptID the task attempt id. Optional.
231       * @return the LogParams
232       * @throws IOException
233       * @throws InterruptedException
234       */
235      public LogParams getLogParams(JobID jobID, TaskAttemptID taskAttemptID)
236          throws IOException, InterruptedException {
237        return client.getLogFileParams(jobID, taskAttemptID);
238      }
239    
240      /**
241       * Get current cluster status.
242       * 
243       * @return object of {@link ClusterMetrics}
244       * @throws IOException
245       * @throws InterruptedException
246       */
247      public ClusterMetrics getClusterStatus() throws IOException, InterruptedException {
248        return client.getClusterMetrics();
249      }
250      
251      /**
252       * Get all active trackers in the cluster.
253       * 
254       * @return array of {@link TaskTrackerInfo}
255       * @throws IOException
256       * @throws InterruptedException
257       */
258      public TaskTrackerInfo[] getActiveTaskTrackers() 
259          throws IOException, InterruptedException  {
260        return client.getActiveTrackers();
261      }
262      
263      /**
264       * Get blacklisted trackers.
265       * 
266       * @return array of {@link TaskTrackerInfo}
267       * @throws IOException
268       * @throws InterruptedException
269       */
270      public TaskTrackerInfo[] getBlackListedTaskTrackers() 
271          throws IOException, InterruptedException  {
272        return client.getBlacklistedTrackers();
273      }
274      
275      /**
276       * Get all the jobs in cluster.
277       * 
278       * @return array of {@link Job}
279       * @throws IOException
280       * @throws InterruptedException
281       * @deprecated Use {@link #getAllJobStatuses()} instead.
282       */
283      @Deprecated
284      public Job[] getAllJobs() throws IOException, InterruptedException {
285        return getJobs(client.getAllJobs());
286      }
287    
288      /**
289       * Get job status for all jobs in the cluster.
290       * @return job status for all jobs in cluster
291       * @throws IOException
292       * @throws InterruptedException
293       */
294      public JobStatus[] getAllJobStatuses() throws IOException, InterruptedException {
295        return client.getAllJobs();
296      }
297    
298      /**
299       * Grab the jobtracker system directory path where 
300       * job-specific files will  be placed.
301       * 
302       * @return the system directory where job-specific files are to be placed.
303       */
304      public Path getSystemDir() throws IOException, InterruptedException {
305        if (sysDir == null) {
306          sysDir = new Path(client.getSystemDir());
307        }
308        return sysDir;
309      }
310      
311      /**
312       * Grab the jobtracker's view of the staging directory path where 
313       * job-specific files will  be placed.
314       * 
315       * @return the staging directory where job-specific files are to be placed.
316       */
317      public Path getStagingAreaDir() throws IOException, InterruptedException {
318        if (stagingAreaDir == null) {
319          stagingAreaDir = new Path(client.getStagingAreaDir());
320        }
321        return stagingAreaDir;
322      }
323    
324      /**
325       * Get the job history file path for a given job id. The job history file at 
326       * this path may or may not be existing depending on the job completion state.
327       * The file is present only for the completed jobs.
328       * @param jobId the JobID of the job submitted by the current user.
329       * @return the file path of the job history file
330       * @throws IOException
331       * @throws InterruptedException
332       */
333      public String getJobHistoryUrl(JobID jobId) throws IOException, 
334        InterruptedException {
335        if (jobHistoryDir == null) {
336          jobHistoryDir = new Path(client.getJobHistoryDir());
337        }
338        return new Path(jobHistoryDir, jobId.toString() + "_"
339                        + ugi.getShortUserName()).toString();
340      }
341    
342      /**
343       * Gets the Queue ACLs for current user
344       * @return array of QueueAclsInfo object for current user.
345       * @throws IOException
346       */
347      public QueueAclsInfo[] getQueueAclsForCurrentUser() 
348          throws IOException, InterruptedException  {
349        return client.getQueueAclsForCurrentUser();
350      }
351    
352      /**
353       * Gets the root level queues.
354       * @return array of JobQueueInfo object.
355       * @throws IOException
356       */
357      public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
358        return client.getRootQueues();
359      }
360      
361      /**
362       * Returns immediate children of queueName.
363       * @param queueName
364       * @return array of JobQueueInfo which are children of queueName
365       * @throws IOException
366       */
367      public QueueInfo[] getChildQueues(String queueName) 
368          throws IOException, InterruptedException {
369        return client.getChildQueues(queueName);
370      }
371      
372      /**
373       * Get the JobTracker's status.
374       * 
375       * @return {@link JobTrackerStatus} of the JobTracker
376       * @throws IOException
377       * @throws InterruptedException
378       */
379      public JobTrackerStatus getJobTrackerStatus() throws IOException,
380          InterruptedException {
381        return client.getJobTrackerStatus();
382      }
383      
384      /**
385       * Get the tasktracker expiry interval for the cluster
386       * @return the expiry interval in msec
387       */
388      public long getTaskTrackerExpiryInterval() throws IOException,
389          InterruptedException {
390        return client.getTaskTrackerExpiryInterval();
391      }
392    
393      /**
394       * Get a delegation token for the user from the JobTracker.
395       * @param renewer the user who can renew the token
396       * @return the new token
397       * @throws IOException
398       */
399      public Token<DelegationTokenIdentifier> 
400          getDelegationToken(Text renewer) throws IOException, InterruptedException{
401        // client has already set the service
402        return client.getDelegationToken(renewer);
403      }
404    
405      /**
406       * Renew a delegation token
407       * @param token the token to renew
408       * @return the new expiration time
409       * @throws InvalidToken
410       * @throws IOException
411       * @deprecated Use {@link Token#renew} instead
412       */
413      public long renewDelegationToken(Token<DelegationTokenIdentifier> token
414                                       ) throws InvalidToken, IOException,
415                                                InterruptedException {
416        try {
417          return client.renewDelegationToken(token);
418        } catch (RemoteException re) {
419          throw re.unwrapRemoteException(InvalidToken.class, 
420                                         AccessControlException.class);
421        }
422      }
423    
424      /**
425       * Cancel a delegation token from the JobTracker
426       * @param token the token to cancel
427       * @throws IOException
428       * @deprecated Use {@link Token#cancel} instead
429       */
430      public void cancelDelegationToken(Token<DelegationTokenIdentifier> token
431                                        ) throws IOException,
432                                                 InterruptedException {
433        try {
434          client.cancelDelegationToken(token);
435        } catch (RemoteException re) {
436          throw re.unwrapRemoteException(InvalidToken.class,
437                                         AccessControlException.class);
438        }
439      }
440    
441    }