001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.conf;
020    
021    import java.io.BufferedInputStream;
022    import java.io.DataInput;
023    import java.io.DataOutput;
024    import java.io.File;
025    import java.io.FileInputStream;
026    import java.io.IOException;
027    import java.io.InputStream;
028    import java.io.InputStreamReader;
029    import java.io.OutputStream;
030    import java.io.OutputStreamWriter;
031    import java.io.Reader;
032    import java.io.Writer;
033    import java.lang.ref.WeakReference;
034    import java.net.InetSocketAddress;
035    import java.net.URL;
036    import java.util.ArrayList;
037    import java.util.Arrays;
038    import java.util.Collection;
039    import java.util.Collections;
040    import java.util.Enumeration;
041    import java.util.HashMap;
042    import java.util.HashSet;
043    import java.util.Iterator;
044    import java.util.LinkedList;
045    import java.util.List;
046    import java.util.ListIterator;
047    import java.util.Map;
048    import java.util.Map.Entry;
049    import java.util.Properties;
050    import java.util.Set;
051    import java.util.StringTokenizer;
052    import java.util.WeakHashMap;
053    import java.util.concurrent.CopyOnWriteArrayList;
054    import java.util.regex.Matcher;
055    import java.util.regex.Pattern;
056    import java.util.regex.PatternSyntaxException;
057    import java.util.concurrent.TimeUnit;
058    import java.util.concurrent.atomic.AtomicBoolean;
059    import java.util.concurrent.atomic.AtomicReference;
060    
061    import javax.xml.parsers.DocumentBuilder;
062    import javax.xml.parsers.DocumentBuilderFactory;
063    import javax.xml.parsers.ParserConfigurationException;
064    import javax.xml.transform.Transformer;
065    import javax.xml.transform.TransformerException;
066    import javax.xml.transform.TransformerFactory;
067    import javax.xml.transform.dom.DOMSource;
068    import javax.xml.transform.stream.StreamResult;
069    
070    import org.apache.commons.collections.map.UnmodifiableMap;
071    import org.apache.commons.logging.Log;
072    import org.apache.commons.logging.LogFactory;
073    import org.apache.hadoop.classification.InterfaceAudience;
074    import org.apache.hadoop.classification.InterfaceStability;
075    import org.apache.hadoop.fs.FileSystem;
076    import org.apache.hadoop.fs.Path;
077    import org.apache.hadoop.fs.CommonConfigurationKeys;
078    import org.apache.hadoop.io.Writable;
079    import org.apache.hadoop.io.WritableUtils;
080    import org.apache.hadoop.net.NetUtils;
081    import org.apache.hadoop.util.ReflectionUtils;
082    import org.apache.hadoop.util.StringInterner;
083    import org.apache.hadoop.util.StringUtils;
084    import org.codehaus.jackson.JsonFactory;
085    import org.codehaus.jackson.JsonGenerator;
086    import org.w3c.dom.DOMException;
087    import org.w3c.dom.Document;
088    import org.w3c.dom.Element;
089    import org.w3c.dom.Node;
090    import org.w3c.dom.NodeList;
091    import org.w3c.dom.Text;
092    import org.xml.sax.SAXException;
093    import com.google.common.base.Preconditions;
094    
095    /** 
096     * Provides access to configuration parameters.
097     *
098     * <h4 id="Resources">Resources</h4>
099     *
100     * <p>Configurations are specified by resources. A resource contains a set of
101     * name/value pairs as XML data. Each resource is named by either a 
102     * <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
103     * then the classpath is examined for a file with that name.  If named by a 
104     * <code>Path</code>, then the local filesystem is examined directly, without 
105     * referring to the classpath.
106     *
107     * <p>Unless explicitly turned off, Hadoop by default specifies two 
108     * resources, loaded in-order from the classpath: <ol>
109     * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
110     * </tt>: Read-only defaults for hadoop.</li>
111     * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
112     * installation.</li>
113     * </ol>
114     * Applications may add additional resources, which are loaded
115     * subsequent to these resources in the order they are added.
116     * 
117     * <h4 id="FinalParams">Final Parameters</h4>
118     *
119     * <p>Configuration parameters may be declared <i>final</i>. 
120     * Once a resource declares a value final, no subsequently-loaded 
121     * resource can alter that value.  
122     * For example, one might define a final parameter with:
123     * <tt><pre>
124     *  &lt;property&gt;
125     *    &lt;name&gt;dfs.client.buffer.dir&lt;/name&gt;
126     *    &lt;value&gt;/tmp/hadoop/dfs/client&lt;/value&gt;
127     *    <b>&lt;final&gt;true&lt;/final&gt;</b>
128     *  &lt;/property&gt;</pre></tt>
129     *
130     * Administrators typically define parameters as final in 
131     * <tt>core-site.xml</tt> for values that user applications may not alter.
132     *
133     * <h4 id="VariableExpansion">Variable Expansion</h4>
134     *
135     * <p>Value strings are first processed for <i>variable expansion</i>. The
136     * available properties are:<ol>
137     * <li>Other properties defined in this Configuration; and, if a name is
138     * undefined here,</li>
139     * <li>Properties in {@link System#getProperties()}.</li>
140     * </ol>
141     *
142     * <p>For example, if a configuration resource contains the following property
143     * definitions: 
144     * <tt><pre>
145     *  &lt;property&gt;
146     *    &lt;name&gt;basedir&lt;/name&gt;
147     *    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
148     *  &lt;/property&gt;
149     *  
150     *  &lt;property&gt;
151     *    &lt;name&gt;tempdir&lt;/name&gt;
152     *    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
153     *  &lt;/property&gt;</pre></tt>
154     *
155     * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
156     * will be resolved to another property in this Configuration, while
157     * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
158     * of the System property with that name.
159     */
160    @InterfaceAudience.Public
161    @InterfaceStability.Stable
162    public class Configuration implements Iterable<Map.Entry<String,String>>,
163                                          Writable {
164      private static final Log LOG =
165        LogFactory.getLog(Configuration.class);
166    
167      private boolean quietmode = true;
168      
169      private static class Resource {
170        private final Object resource;
171        private final String name;
172        
173        public Resource(Object resource) {
174          this(resource, resource.toString());
175        }
176        
177        public Resource(Object resource, String name) {
178          this.resource = resource;
179          this.name = name;
180        }
181        
182        public String getName(){
183          return name;
184        }
185        
186        public Object getResource() {
187          return resource;
188        }
189        
190        @Override
191        public String toString() {
192          return name;
193        }
194      }
195      
196      /**
197       * List of configuration resources.
198       */
199      private ArrayList<Resource> resources = new ArrayList<Resource>();
200      
201      /**
202       * The value reported as the setting resource when a key is set
203       * by code rather than a file resource by dumpConfiguration.
204       */
205      static final String UNKNOWN_RESOURCE = "Unknown";
206    
207    
208      /**
209       * List of configuration parameters marked <b>final</b>. 
210       */
211      private Set<String> finalParameters = new HashSet<String>();
212      
213      private boolean loadDefaults = true;
214      
215      /**
216       * Configuration objects
217       */
218      private static final WeakHashMap<Configuration,Object> REGISTRY = 
219        new WeakHashMap<Configuration,Object>();
220      
221      /**
222       * List of default Resources. Resources are loaded in the order of the list 
223       * entries
224       */
225      private static final CopyOnWriteArrayList<String> defaultResources =
226        new CopyOnWriteArrayList<String>();
227    
228      private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
229        CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
230    
231      /**
232       * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
233       */
234      private static final Class<?> NEGATIVE_CACHE_SENTINEL =
235        NegativeCacheSentinel.class;
236    
237      /**
238       * Stores the mapping of key to the resource which modifies or loads 
239       * the key most recently
240       */
241      private HashMap<String, String[]> updatingResource;
242     
243      /**
244       * Class to keep the information about the keys which replace the deprecated
245       * ones.
246       * 
247       * This class stores the new keys which replace the deprecated keys and also
248       * gives a provision to have a custom message for each of the deprecated key
249       * that is being replaced. It also provides method to get the appropriate
250       * warning message which can be logged whenever the deprecated key is used.
251       */
252      private static class DeprecatedKeyInfo {
253        private final String[] newKeys;
254        private final String customMessage;
255        private final AtomicBoolean accessed = new AtomicBoolean(false);
256    
257        DeprecatedKeyInfo(String[] newKeys, String customMessage) {
258          this.newKeys = newKeys;
259          this.customMessage = customMessage;
260        }
261    
262        /**
263         * Method to provide the warning message. It gives the custom message if
264         * non-null, and default message otherwise.
265         * @param key the associated deprecated key.
266         * @return message that is to be logged when a deprecated key is used.
267         */
268        private final String getWarningMessage(String key) {
269          String warningMessage;
270          if(customMessage == null) {
271            StringBuilder message = new StringBuilder(key);
272            String deprecatedKeySuffix = " is deprecated. Instead, use ";
273            message.append(deprecatedKeySuffix);
274            for (int i = 0; i < newKeys.length; i++) {
275              message.append(newKeys[i]);
276              if(i != newKeys.length-1) {
277                message.append(", ");
278              }
279            }
280            warningMessage = message.toString();
281          }
282          else {
283            warningMessage = customMessage;
284          }
285          return warningMessage;
286        }
287    
288        boolean getAndSetAccessed() {
289          return accessed.getAndSet(true);
290        }
291    
292        public void clearAccessed() {
293          accessed.set(false);
294        }
295      }
296      
297      /**
298       * A pending addition to the global set of deprecated keys.
299       */
300      public static class DeprecationDelta {
301        private final String key;
302        private final String[] newKeys;
303        private final String customMessage;
304    
305        DeprecationDelta(String key, String[] newKeys, String customMessage) {
306          Preconditions.checkNotNull(key);
307          Preconditions.checkNotNull(newKeys);
308          Preconditions.checkArgument(newKeys.length > 0);
309          this.key = key;
310          this.newKeys = newKeys;
311          this.customMessage = customMessage;
312        }
313    
314        public DeprecationDelta(String key, String newKey, String customMessage) {
315          this(key, new String[] { newKey }, customMessage);
316        }
317    
318        public DeprecationDelta(String key, String newKey) {
319          this(key, new String[] { newKey }, null);
320        }
321    
322        public String getKey() {
323          return key;
324        }
325    
326        public String[] getNewKeys() {
327          return newKeys;
328        }
329    
330        public String getCustomMessage() {
331          return customMessage;
332        }
333      }
334    
335      /**
336       * The set of all keys which are deprecated.
337       *
338       * DeprecationContext objects are immutable.
339       */
340      private static class DeprecationContext {
341        /**
342         * Stores the deprecated keys, the new keys which replace the deprecated keys
343         * and custom message(if any provided).
344         */
345        private final Map<String, DeprecatedKeyInfo> deprecatedKeyMap;
346    
347        /**
348         * Stores a mapping from superseding keys to the keys which they deprecate.
349         */
350        private final Map<String, String> reverseDeprecatedKeyMap;
351    
352        /**
353         * Create a new DeprecationContext by copying a previous DeprecationContext
354         * and adding some deltas.
355         *
356         * @param other   The previous deprecation context to copy, or null to start
357         *                from nothing.
358         * @param deltas  The deltas to apply.
359         */
360        @SuppressWarnings("unchecked")
361        DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
362          HashMap<String, DeprecatedKeyInfo> newDeprecatedKeyMap = 
363            new HashMap<String, DeprecatedKeyInfo>();
364          HashMap<String, String> newReverseDeprecatedKeyMap =
365            new HashMap<String, String>();
366          if (other != null) {
367            for (Entry<String, DeprecatedKeyInfo> entry :
368                other.deprecatedKeyMap.entrySet()) {
369              newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
370            }
371            for (Entry<String, String> entry :
372                other.reverseDeprecatedKeyMap.entrySet()) {
373              newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
374            }
375          }
376          for (DeprecationDelta delta : deltas) {
377            if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
378              DeprecatedKeyInfo newKeyInfo =
379                new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
380              newDeprecatedKeyMap.put(delta.key, newKeyInfo);
381              for (String newKey : delta.getNewKeys()) {
382                newReverseDeprecatedKeyMap.put(newKey, delta.key);
383              }
384            }
385          }
386          this.deprecatedKeyMap =
387            UnmodifiableMap.decorate(newDeprecatedKeyMap);
388          this.reverseDeprecatedKeyMap =
389            UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
390        }
391    
392        Map<String, DeprecatedKeyInfo> getDeprecatedKeyMap() {
393          return deprecatedKeyMap;
394        }
395    
396        Map<String, String> getReverseDeprecatedKeyMap() {
397          return reverseDeprecatedKeyMap;
398        }
399      }
400      
401      private static DeprecationDelta[] defaultDeprecations = 
402        new DeprecationDelta[] {
403          new DeprecationDelta("topology.script.file.name", 
404            CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
405          new DeprecationDelta("topology.script.number.args", 
406            CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
407          new DeprecationDelta("hadoop.configured.node.mapping", 
408            CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
409          new DeprecationDelta("topology.node.switch.mapping.impl", 
410            CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
411          new DeprecationDelta("dfs.df.interval", 
412            CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
413          new DeprecationDelta("dfs.client.buffer.dir", 
414            CommonConfigurationKeys.FS_CLIENT_BUFFER_DIR_KEY),
415          new DeprecationDelta("hadoop.native.lib", 
416            CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY),
417          new DeprecationDelta("fs.default.name", 
418            CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
419          new DeprecationDelta("dfs.umaskmode",
420            CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY)
421        };
422    
423      /**
424       * The global DeprecationContext.
425       */
426      private static AtomicReference<DeprecationContext> deprecationContext =
427          new AtomicReference<DeprecationContext>(
428              new DeprecationContext(null, defaultDeprecations));
429    
430      /**
431       * Adds a set of deprecated keys to the global deprecations.
432       *
433       * This method is lockless.  It works by means of creating a new
434       * DeprecationContext based on the old one, and then atomically swapping in
435       * the new context.  If someone else updated the context in between us reading
436       * the old context and swapping in the new one, we try again until we win the
437       * race.
438       *
439       * @param deltas   The deprecations to add.
440       */
441      public static void addDeprecations(DeprecationDelta[] deltas) {
442        DeprecationContext prev, next;
443        do {
444          prev = deprecationContext.get();
445          next = new DeprecationContext(prev, deltas);
446        } while (!deprecationContext.compareAndSet(prev, next));
447      }
448    
449      /**
450       * Adds the deprecated key to the global deprecation map.
451       * It does not override any existing entries in the deprecation map.
452       * This is to be used only by the developers in order to add deprecation of
453       * keys, and attempts to call this method after loading resources once,
454       * would lead to <tt>UnsupportedOperationException</tt>
455       * 
456       * If a key is deprecated in favor of multiple keys, they are all treated as 
457       * aliases of each other, and setting any one of them resets all the others 
458       * to the new value.
459       *
460       * If you have multiple deprecation entries to add, it is more efficient to
461       * use #addDeprecations(DeprecationDelta[] deltas) instead.
462       * 
463       * @param key
464       * @param newKeys
465       * @param customMessage
466       * @deprecated use {@link #addDeprecation(String key, String newKey,
467          String customMessage)} instead
468       */
469      @Deprecated
470      public static void addDeprecation(String key, String[] newKeys,
471          String customMessage) {
472        addDeprecations(new DeprecationDelta[] {
473          new DeprecationDelta(key, newKeys, customMessage)
474        });
475      }
476    
477      /**
478       * Adds the deprecated key to the global deprecation map.
479       * It does not override any existing entries in the deprecation map.
480       * This is to be used only by the developers in order to add deprecation of
481       * keys, and attempts to call this method after loading resources once,
482       * would lead to <tt>UnsupportedOperationException</tt>
483       * 
484       * If you have multiple deprecation entries to add, it is more efficient to
485       * use #addDeprecations(DeprecationDelta[] deltas) instead.
486       *
487       * @param key
488       * @param newKey
489       * @param customMessage
490       */
491      public static void addDeprecation(String key, String newKey,
492                  String customMessage) {
493              addDeprecation(key, new String[] {newKey}, customMessage);
494      }
495    
496      /**
497       * Adds the deprecated key to the global deprecation map when no custom
498       * message is provided.
499       * It does not override any existing entries in the deprecation map.
500       * This is to be used only by the developers in order to add deprecation of
501       * keys, and attempts to call this method after loading resources once,
502       * would lead to <tt>UnsupportedOperationException</tt>
503       * 
504       * If a key is deprecated in favor of multiple keys, they are all treated as 
505       * aliases of each other, and setting any one of them resets all the others 
506       * to the new value.
507       * 
508       * If you have multiple deprecation entries to add, it is more efficient to
509       * use #addDeprecations(DeprecationDelta[] deltas) instead.
510       *
511       * @param key Key that is to be deprecated
512       * @param newKeys list of keys that take up the values of deprecated key
513       * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
514       */
515      @Deprecated
516      public static void addDeprecation(String key, String[] newKeys) {
517        addDeprecation(key, newKeys, null);
518      }
519      
520      /**
521       * Adds the deprecated key to the global deprecation map when no custom
522       * message is provided.
523       * It does not override any existing entries in the deprecation map.
524       * This is to be used only by the developers in order to add deprecation of
525       * keys, and attempts to call this method after loading resources once,
526       * would lead to <tt>UnsupportedOperationException</tt>
527       * 
528       * If you have multiple deprecation entries to add, it is more efficient to
529       * use #addDeprecations(DeprecationDelta[] deltas) instead.
530       *
531       * @param key Key that is to be deprecated
532       * @param newKey key that takes up the value of deprecated key
533       */
534      public static void addDeprecation(String key, String newKey) {
535        addDeprecation(key, new String[] {newKey}, null);
536      }
537      
538      /**
539       * checks whether the given <code>key</code> is deprecated.
540       * 
541       * @param key the parameter which is to be checked for deprecation
542       * @return <code>true</code> if the key is deprecated and 
543       *         <code>false</code> otherwise.
544       */
545      public static boolean isDeprecated(String key) {
546        return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
547      }
548    
549      /**
550       * Returns the alternate name for a key if the property name is deprecated
551       * or if deprecates a property name.
552       *
553       * @param name property name.
554       * @return alternate name.
555       */
556      private String[] getAlternateNames(String name) {
557        String altNames[] = null;
558        DeprecationContext cur = deprecationContext.get();
559        DeprecatedKeyInfo keyInfo = cur.getDeprecatedKeyMap().get(name);
560        if (keyInfo == null) {
561          altNames = (cur.getReverseDeprecatedKeyMap().get(name) != null ) ? 
562            new String [] {cur.getReverseDeprecatedKeyMap().get(name)} : null;
563          if(altNames != null && altNames.length > 0) {
564            //To help look for other new configs for this deprecated config
565            keyInfo = cur.getDeprecatedKeyMap().get(altNames[0]);
566          }      
567        } 
568        if(keyInfo != null && keyInfo.newKeys.length > 0) {
569          List<String> list = new ArrayList<String>(); 
570          if(altNames != null) {
571              list.addAll(Arrays.asList(altNames));
572          }
573          list.addAll(Arrays.asList(keyInfo.newKeys));
574          altNames = list.toArray(new String[list.size()]);
575        }
576        return altNames;
577      }
578    
579      /**
580       * Checks for the presence of the property <code>name</code> in the
581       * deprecation map. Returns the first of the list of new keys if present
582       * in the deprecation map or the <code>name</code> itself. If the property
583       * is not presently set but the property map contains an entry for the
584       * deprecated key, the value of the deprecated key is set as the value for
585       * the provided property name.
586       *
587       * @param name the property name
588       * @return the first property in the list of properties mapping
589       *         the <code>name</code> or the <code>name</code> itself.
590       */
591      private String[] handleDeprecation(DeprecationContext deprecations,
592          String name) {
593        ArrayList<String > names = new ArrayList<String>();
594            if (isDeprecated(name)) {
595          DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
596          warnOnceIfDeprecated(deprecations, name);
597          for (String newKey : keyInfo.newKeys) {
598            if(newKey != null) {
599              names.add(newKey);
600            }
601          }
602        }
603        if(names.size() == 0) {
604            names.add(name);
605        }
606        for(String n : names) {
607              String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
608              if (deprecatedKey != null && !getOverlay().containsKey(n) &&
609                  getOverlay().containsKey(deprecatedKey)) {
610                getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
611                getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
612              }
613        }
614        return names.toArray(new String[names.size()]);
615      }
616     
617      private void handleDeprecation() {
618        LOG.debug("Handling deprecation for all properties in config...");
619        DeprecationContext deprecations = deprecationContext.get();
620        Set<Object> keys = new HashSet<Object>();
621        keys.addAll(getProps().keySet());
622        for (Object item: keys) {
623          LOG.debug("Handling deprecation for " + (String)item);
624          handleDeprecation(deprecations, (String)item);
625        }
626      }
627     
628      static{
629        //print deprecation warning if hadoop-site.xml is found in classpath
630        ClassLoader cL = Thread.currentThread().getContextClassLoader();
631        if (cL == null) {
632          cL = Configuration.class.getClassLoader();
633        }
634        if(cL.getResource("hadoop-site.xml")!=null) {
635          LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
636              "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
637              + "mapred-site.xml and hdfs-site.xml to override properties of " +
638              "core-default.xml, mapred-default.xml and hdfs-default.xml " +
639              "respectively");
640        }
641        addDefaultResource("core-default.xml");
642        addDefaultResource("core-site.xml");
643      }
644      
645      private Properties properties;
646      private Properties overlay;
647      private ClassLoader classLoader;
648      {
649        classLoader = Thread.currentThread().getContextClassLoader();
650        if (classLoader == null) {
651          classLoader = Configuration.class.getClassLoader();
652        }
653      }
654      
655      /** A new configuration. */
656      public Configuration() {
657        this(true);
658      }
659    
660      /** A new configuration where the behavior of reading from the default 
661       * resources can be turned off.
662       * 
663       * If the parameter {@code loadDefaults} is false, the new instance
664       * will not load resources from the default files. 
665       * @param loadDefaults specifies whether to load from the default files
666       */
667      public Configuration(boolean loadDefaults) {
668        this.loadDefaults = loadDefaults;
669        updatingResource = new HashMap<String, String[]>();
670        synchronized(Configuration.class) {
671          REGISTRY.put(this, null);
672        }
673      }
674      
675      /** 
676       * A new configuration with the same settings cloned from another.
677       * 
678       * @param other the configuration from which to clone settings.
679       */
680      @SuppressWarnings("unchecked")
681      public Configuration(Configuration other) {
682       this.resources = (ArrayList<Resource>) other.resources.clone();
683       synchronized(other) {
684         if (other.properties != null) {
685           this.properties = (Properties)other.properties.clone();
686         }
687    
688         if (other.overlay!=null) {
689           this.overlay = (Properties)other.overlay.clone();
690         }
691    
692         this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
693       }
694       
695        this.finalParameters = new HashSet<String>(other.finalParameters);
696        synchronized(Configuration.class) {
697          REGISTRY.put(this, null);
698        }
699        this.classLoader = other.classLoader;
700        this.loadDefaults = other.loadDefaults;
701        setQuietMode(other.getQuietMode());
702      }
703      
704      /**
705       * Add a default resource. Resources are loaded in the order of the resources 
706       * added.
707       * @param name file name. File should be present in the classpath.
708       */
709      public static synchronized void addDefaultResource(String name) {
710        if(!defaultResources.contains(name)) {
711          defaultResources.add(name);
712          for(Configuration conf : REGISTRY.keySet()) {
713            if(conf.loadDefaults) {
714              conf.reloadConfiguration();
715            }
716          }
717        }
718      }
719    
720      /**
721       * Add a configuration resource. 
722       * 
723       * The properties of this resource will override properties of previously 
724       * added resources, unless they were marked <a href="#Final">final</a>. 
725       * 
726       * @param name resource to be added, the classpath is examined for a file 
727       *             with that name.
728       */
729      public void addResource(String name) {
730        addResourceObject(new Resource(name));
731      }
732    
733      /**
734       * Add a configuration resource. 
735       * 
736       * The properties of this resource will override properties of previously 
737       * added resources, unless they were marked <a href="#Final">final</a>. 
738       * 
739       * @param url url of the resource to be added, the local filesystem is 
740       *            examined directly to find the resource, without referring to 
741       *            the classpath.
742       */
743      public void addResource(URL url) {
744        addResourceObject(new Resource(url));
745      }
746    
747      /**
748       * Add a configuration resource. 
749       * 
750       * The properties of this resource will override properties of previously 
751       * added resources, unless they were marked <a href="#Final">final</a>. 
752       * 
753       * @param file file-path of resource to be added, the local filesystem is
754       *             examined directly to find the resource, without referring to 
755       *             the classpath.
756       */
757      public void addResource(Path file) {
758        addResourceObject(new Resource(file));
759      }
760    
761      /**
762       * Add a configuration resource. 
763       * 
764       * The properties of this resource will override properties of previously 
765       * added resources, unless they were marked <a href="#Final">final</a>. 
766       * 
767       * WARNING: The contents of the InputStream will be cached, by this method. 
768       * So use this sparingly because it does increase the memory consumption.
769       * 
770       * @param in InputStream to deserialize the object from. In will be read from
771       * when a get or set is called next.  After it is read the stream will be
772       * closed. 
773       */
774      public void addResource(InputStream in) {
775        addResourceObject(new Resource(in));
776      }
777    
778      /**
779       * Add a configuration resource. 
780       * 
781       * The properties of this resource will override properties of previously 
782       * added resources, unless they were marked <a href="#Final">final</a>. 
783       * 
784       * @param in InputStream to deserialize the object from.
785       * @param name the name of the resource because InputStream.toString is not
786       * very descriptive some times.  
787       */
788      public void addResource(InputStream in, String name) {
789        addResourceObject(new Resource(in, name));
790      }
791      
792      
793      /**
794       * Reload configuration from previously added resources.
795       *
796       * This method will clear all the configuration read from the added 
797       * resources, and final parameters. This will make the resources to 
798       * be read again before accessing the values. Values that are added
799       * via set methods will overlay values read from the resources.
800       */
801      public synchronized void reloadConfiguration() {
802        properties = null;                            // trigger reload
803        finalParameters.clear();                      // clear site-limits
804      }
805      
806      private synchronized void addResourceObject(Resource resource) {
807        resources.add(resource);                      // add to resources
808        reloadConfiguration();
809      }
810      
811      private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
812      private static int MAX_SUBST = 20;
813    
814      private String substituteVars(String expr) {
815        if (expr == null) {
816          return null;
817        }
818        Matcher match = varPat.matcher("");
819        String eval = expr;
820        for(int s=0; s<MAX_SUBST; s++) {
821          match.reset(eval);
822          if (!match.find()) {
823            return eval;
824          }
825          String var = match.group();
826          var = var.substring(2, var.length()-1); // remove ${ .. }
827          String val = null;
828          try {
829            val = System.getProperty(var);
830          } catch(SecurityException se) {
831            LOG.warn("Unexpected SecurityException in Configuration", se);
832          }
833          if (val == null) {
834            val = getRaw(var);
835          }
836          if (val == null) {
837            return eval; // return literal ${var}: var is unbound
838          }
839          // substitute
840          eval = eval.substring(0, match.start())+val+eval.substring(match.end());
841        }
842        throw new IllegalStateException("Variable substitution depth too large: " 
843                                        + MAX_SUBST + " " + expr);
844      }
845      
846      /**
847       * Get the value of the <code>name</code> property, <code>null</code> if
848       * no such property exists. If the key is deprecated, it returns the value of
849       * the first key which replaces the deprecated key and is not null
850       * 
851       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
852       * before being returned. 
853       * 
854       * @param name the property name.
855       * @return the value of the <code>name</code> or its replacing property, 
856       *         or null if no such property exists.
857       */
858      public String get(String name) {
859        String[] names = handleDeprecation(deprecationContext.get(), name);
860        String result = null;
861        for(String n : names) {
862          result = substituteVars(getProps().getProperty(n));
863        }
864        return result;
865      }
866      
867      /**
868       * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
869       * <code>null</code> if no such property exists. 
870       * If the key is deprecated, it returns the value of
871       * the first key which replaces the deprecated key and is not null
872       * 
873       * Values are processed for <a href="#VariableExpansion">variable expansion</a> 
874       * before being returned. 
875       * 
876       * @param name the property name.
877       * @return the value of the <code>name</code> or its replacing property, 
878       *         or null if no such property exists.
879       */
880      public String getTrimmed(String name) {
881        String value = get(name);
882        
883        if (null == value) {
884          return null;
885        } else {
886          return value.trim();
887        }
888      }
889      
890      /**
891       * Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
892       * <code>defaultVal</code> if no such property exists. 
893       * See @{Configuration#getTrimmed} for more details.
894       * 
895       * @param name          the property name.
896       * @param defaultVal    the property default value.
897       * @return              the value of the <code>name</code> or defaultVal 
898       *                      if it is not set.
899       */
900      public String getTrimmed(String name, String defaultValue) {
901        String ret = getTrimmed(name);
902        return ret == null ? defaultValue : ret;
903      }
904    
905      /**
906       * Get the value of the <code>name</code> property, without doing
907       * <a href="#VariableExpansion">variable expansion</a>.If the key is 
908       * deprecated, it returns the value of the first key which replaces 
909       * the deprecated key and is not null.
910       * 
911       * @param name the property name.
912       * @return the value of the <code>name</code> property or 
913       *         its replacing property and null if no such property exists.
914       */
915      public String getRaw(String name) {
916        String[] names = handleDeprecation(deprecationContext.get(), name);
917        String result = null;
918        for(String n : names) {
919          result = getProps().getProperty(n);
920        }
921        return result;
922      }
923    
924      /** 
925       * Set the <code>value</code> of the <code>name</code> property. If 
926       * <code>name</code> is deprecated or there is a deprecated name associated to it,
927       * it sets the value to both names.
928       * 
929       * @param name property name.
930       * @param value property value.
931       */
932      public void set(String name, String value) {
933        set(name, value, null);
934      }
935      
936      /** 
937       * Set the <code>value</code> of the <code>name</code> property. If 
938       * <code>name</code> is deprecated or there is a deprecated name associated to it,
939       * it sets the value to both names.
940       * 
941       * @param name property name.
942       * @param value property value.
943       * @param source the place that this configuration value came from 
944       * (For debugging).
945       */
946      public void set(String name, String value, String source) {
947        Preconditions.checkArgument(
948            name != null,
949            "Property name must not be null");
950        Preconditions.checkArgument(
951            value != null,
952            "Property value must not be null");
953        DeprecationContext deprecations = deprecationContext.get();
954        if (deprecations.getDeprecatedKeyMap().isEmpty()) {
955          getProps();
956        }
957        getOverlay().setProperty(name, value);
958        getProps().setProperty(name, value);
959        if(source == null) {
960          updatingResource.put(name, new String[] {"programatically"});
961        } else {
962          updatingResource.put(name, new String[] {source});
963        }
964        String[] altNames = getAlternateNames(name);
965        if (altNames != null && altNames.length > 0) {
966          String altSource = "because " + name + " is deprecated";
967          for(String altName : altNames) {
968            if(!altName.equals(name)) {
969              getOverlay().setProperty(altName, value);
970              getProps().setProperty(altName, value);
971              updatingResource.put(altName, new String[] {altSource});
972            }
973          }
974        }
975        warnOnceIfDeprecated(deprecations, name);
976      }
977    
978      private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
979        DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
980        if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
981          LOG.warn(keyInfo.getWarningMessage(name));
982        }
983      }
984    
985      /**
986       * Unset a previously set property.
987       */
988      public synchronized void unset(String name) {
989        String[] altNames = getAlternateNames(name);
990        getOverlay().remove(name);
991        getProps().remove(name);
992        if (altNames !=null && altNames.length > 0) {
993          for(String altName : altNames) {
994            getOverlay().remove(altName);
995            getProps().remove(altName);
996          }
997        }
998      }
999    
1000      /**
1001       * Sets a property if it is currently unset.
1002       * @param name the property name
1003       * @param value the new value
1004       */
1005      public synchronized void setIfUnset(String name, String value) {
1006        if (get(name) == null) {
1007          set(name, value);
1008        }
1009      }
1010      
1011      private synchronized Properties getOverlay() {
1012        if (overlay==null){
1013          overlay=new Properties();
1014        }
1015        return overlay;
1016      }
1017    
1018      /** 
1019       * Get the value of the <code>name</code>. If the key is deprecated,
1020       * it returns the value of the first key which replaces the deprecated key
1021       * and is not null.
1022       * If no such property exists,
1023       * then <code>defaultValue</code> is returned.
1024       * 
1025       * @param name property name.
1026       * @param defaultValue default value.
1027       * @return property value, or <code>defaultValue</code> if the property 
1028       *         doesn't exist.                    
1029       */
1030      public String get(String name, String defaultValue) {
1031        String[] names = handleDeprecation(deprecationContext.get(), name);
1032        String result = null;
1033        for(String n : names) {
1034          result = substituteVars(getProps().getProperty(n, defaultValue));
1035        }
1036        return result;
1037      }
1038    
1039      /** 
1040       * Get the value of the <code>name</code> property as an <code>int</code>.
1041       *   
1042       * If no such property exists, the provided default value is returned,
1043       * or if the specified value is not a valid <code>int</code>,
1044       * then an error is thrown.
1045       * 
1046       * @param name property name.
1047       * @param defaultValue default value.
1048       * @throws NumberFormatException when the value is invalid
1049       * @return property value as an <code>int</code>, 
1050       *         or <code>defaultValue</code>. 
1051       */
1052      public int getInt(String name, int defaultValue) {
1053        String valueString = getTrimmed(name);
1054        if (valueString == null)
1055          return defaultValue;
1056        String hexString = getHexDigits(valueString);
1057        if (hexString != null) {
1058          return Integer.parseInt(hexString, 16);
1059        }
1060        return Integer.parseInt(valueString);
1061      }
1062      
1063      /**
1064       * Get the value of the <code>name</code> property as a set of comma-delimited
1065       * <code>int</code> values.
1066       * 
1067       * If no such property exists, an empty array is returned.
1068       * 
1069       * @param name property name
1070       * @return property value interpreted as an array of comma-delimited
1071       *         <code>int</code> values
1072       */
1073      public int[] getInts(String name) {
1074        String[] strings = getTrimmedStrings(name);
1075        int[] ints = new int[strings.length];
1076        for (int i = 0; i < strings.length; i++) {
1077          ints[i] = Integer.parseInt(strings[i]);
1078        }
1079        return ints;
1080      }
1081    
1082      /** 
1083       * Set the value of the <code>name</code> property to an <code>int</code>.
1084       * 
1085       * @param name property name.
1086       * @param value <code>int</code> value of the property.
1087       */
1088      public void setInt(String name, int value) {
1089        set(name, Integer.toString(value));
1090      }
1091    
1092    
1093      /** 
1094       * Get the value of the <code>name</code> property as a <code>long</code>.  
1095       * If no such property exists, the provided default value is returned,
1096       * or if the specified value is not a valid <code>long</code>,
1097       * then an error is thrown.
1098       * 
1099       * @param name property name.
1100       * @param defaultValue default value.
1101       * @throws NumberFormatException when the value is invalid
1102       * @return property value as a <code>long</code>, 
1103       *         or <code>defaultValue</code>. 
1104       */
1105      public long getLong(String name, long defaultValue) {
1106        String valueString = getTrimmed(name);
1107        if (valueString == null)
1108          return defaultValue;
1109        String hexString = getHexDigits(valueString);
1110        if (hexString != null) {
1111          return Long.parseLong(hexString, 16);
1112        }
1113        return Long.parseLong(valueString);
1114      }
1115    
1116      /**
1117       * Get the value of the <code>name</code> property as a <code>long</code> or
1118       * human readable format. If no such property exists, the provided default
1119       * value is returned, or if the specified value is not a valid
1120       * <code>long</code> or human readable format, then an error is thrown. You
1121       * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
1122       * t(tera), p(peta), e(exa)
1123       *
1124       * @param name property name.
1125       * @param defaultValue default value.
1126       * @throws NumberFormatException when the value is invalid
1127       * @return property value as a <code>long</code>,
1128       *         or <code>defaultValue</code>.
1129       */
1130      public long getLongBytes(String name, long defaultValue) {
1131        String valueString = getTrimmed(name);
1132        if (valueString == null)
1133          return defaultValue;
1134        return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
1135      }
1136    
1137      private String getHexDigits(String value) {
1138        boolean negative = false;
1139        String str = value;
1140        String hexString = null;
1141        if (value.startsWith("-")) {
1142          negative = true;
1143          str = value.substring(1);
1144        }
1145        if (str.startsWith("0x") || str.startsWith("0X")) {
1146          hexString = str.substring(2);
1147          if (negative) {
1148            hexString = "-" + hexString;
1149          }
1150          return hexString;
1151        }
1152        return null;
1153      }
1154      
1155      /** 
1156       * Set the value of the <code>name</code> property to a <code>long</code>.
1157       * 
1158       * @param name property name.
1159       * @param value <code>long</code> value of the property.
1160       */
1161      public void setLong(String name, long value) {
1162        set(name, Long.toString(value));
1163      }
1164    
1165      /** 
1166       * Get the value of the <code>name</code> property as a <code>float</code>.  
1167       * If no such property exists, the provided default value is returned,
1168       * or if the specified value is not a valid <code>float</code>,
1169       * then an error is thrown.
1170       *
1171       * @param name property name.
1172       * @param defaultValue default value.
1173       * @throws NumberFormatException when the value is invalid
1174       * @return property value as a <code>float</code>, 
1175       *         or <code>defaultValue</code>. 
1176       */
1177      public float getFloat(String name, float defaultValue) {
1178        String valueString = getTrimmed(name);
1179        if (valueString == null)
1180          return defaultValue;
1181        return Float.parseFloat(valueString);
1182      }
1183    
1184      /**
1185       * Set the value of the <code>name</code> property to a <code>float</code>.
1186       * 
1187       * @param name property name.
1188       * @param value property value.
1189       */
1190      public void setFloat(String name, float value) {
1191        set(name,Float.toString(value));
1192      }
1193    
1194      /** 
1195       * Get the value of the <code>name</code> property as a <code>double</code>.  
1196       * If no such property exists, the provided default value is returned,
1197       * or if the specified value is not a valid <code>double</code>,
1198       * then an error is thrown.
1199       *
1200       * @param name property name.
1201       * @param defaultValue default value.
1202       * @throws NumberFormatException when the value is invalid
1203       * @return property value as a <code>double</code>, 
1204       *         or <code>defaultValue</code>. 
1205       */
1206      public double getDouble(String name, double defaultValue) {
1207        String valueString = getTrimmed(name);
1208        if (valueString == null)
1209          return defaultValue;
1210        return Double.parseDouble(valueString);
1211      }
1212    
1213      /**
1214       * Set the value of the <code>name</code> property to a <code>double</code>.
1215       * 
1216       * @param name property name.
1217       * @param value property value.
1218       */
1219      public void setDouble(String name, double value) {
1220        set(name,Double.toString(value));
1221      }
1222     
1223      /** 
1224       * Get the value of the <code>name</code> property as a <code>boolean</code>.  
1225       * If no such property is specified, or if the specified value is not a valid
1226       * <code>boolean</code>, then <code>defaultValue</code> is returned.
1227       * 
1228       * @param name property name.
1229       * @param defaultValue default value.
1230       * @return property value as a <code>boolean</code>, 
1231       *         or <code>defaultValue</code>. 
1232       */
1233      public boolean getBoolean(String name, boolean defaultValue) {
1234        String valueString = getTrimmed(name);
1235        if (null == valueString || "".equals(valueString)) {
1236          return defaultValue;
1237        }
1238    
1239        valueString = valueString.toLowerCase();
1240    
1241        if ("true".equals(valueString))
1242          return true;
1243        else if ("false".equals(valueString))
1244          return false;
1245        else return defaultValue;
1246      }
1247    
1248      /** 
1249       * Set the value of the <code>name</code> property to a <code>boolean</code>.
1250       * 
1251       * @param name property name.
1252       * @param value <code>boolean</code> value of the property.
1253       */
1254      public void setBoolean(String name, boolean value) {
1255        set(name, Boolean.toString(value));
1256      }
1257    
1258      /**
1259       * Set the given property, if it is currently unset.
1260       * @param name property name
1261       * @param value new value
1262       */
1263      public void setBooleanIfUnset(String name, boolean value) {
1264        setIfUnset(name, Boolean.toString(value));
1265      }
1266    
1267      /**
1268       * Set the value of the <code>name</code> property to the given type. This
1269       * is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
1270       * @param name property name
1271       * @param value new value
1272       */
1273      public <T extends Enum<T>> void setEnum(String name, T value) {
1274        set(name, value.toString());
1275      }
1276    
1277      /**
1278       * Return value matching this enumerated type.
1279       * @param name Property name
1280       * @param defaultValue Value returned if no mapping exists
1281       * @throws IllegalArgumentException If mapping is illegal for the type
1282       * provided
1283       */
1284      public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
1285        final String val = get(name);
1286        return null == val
1287          ? defaultValue
1288          : Enum.valueOf(defaultValue.getDeclaringClass(), val);
1289      }
1290    
1291      /**
1292       * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1293       * If no such property is specified, or if the specified value is not a valid
1294       * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1295       *
1296       * @param name property name
1297       * @param defaultValue default value
1298       * @return property value as a compiled Pattern, or defaultValue
1299       */
1300      public Pattern getPattern(String name, Pattern defaultValue) {
1301        String valString = get(name);
1302        if (null == valString || "".equals(valString)) {
1303          return defaultValue;
1304        }
1305        try {
1306          return Pattern.compile(valString);
1307        } catch (PatternSyntaxException pse) {
1308          LOG.warn("Regular expression '" + valString + "' for property '" +
1309                   name + "' not valid. Using default", pse);
1310          return defaultValue;
1311        }
1312      }
1313    
1314      /**
1315       * Set the given property to <code>Pattern</code>.
1316       * If the pattern is passed as null, sets the empty pattern which results in
1317       * further calls to getPattern(...) returning the default value.
1318       *
1319       * @param name property name
1320       * @param pattern new value
1321       */
1322      public void setPattern(String name, Pattern pattern) {
1323        if (null == pattern) {
1324          set(name, null);
1325        } else {
1326          set(name, pattern.pattern());
1327        }
1328      }
1329    
1330      /**
1331       * Gets information about why a property was set.  Typically this is the 
1332       * path to the resource objects (file, URL, etc.) the property came from, but
1333       * it can also indicate that it was set programatically, or because of the
1334       * command line.
1335       *
1336       * @param name - The property name to get the source of.
1337       * @return null - If the property or its source wasn't found. Otherwise, 
1338       * returns a list of the sources of the resource.  The older sources are
1339       * the first ones in the list.  So for example if a configuration is set from
1340       * the command line, and then written out to a file that is read back in the
1341       * first entry would indicate that it was set from the command line, while
1342       * the second one would indicate the file that the new configuration was read
1343       * in from.
1344       */
1345      @InterfaceStability.Unstable
1346      public synchronized String[] getPropertySources(String name) {
1347        if (properties == null) {
1348          // If properties is null, it means a resource was newly added
1349          // but the props were cleared so as to load it upon future
1350          // requests. So lets force a load by asking a properties list.
1351          getProps();
1352        }
1353        // Return a null right away if our properties still
1354        // haven't loaded or the resource mapping isn't defined
1355        if (properties == null || updatingResource == null) {
1356          return null;
1357        } else {
1358          String[] source = updatingResource.get(name);
1359          if(source == null) {
1360            return null;
1361          } else {
1362            return Arrays.copyOf(source, source.length);
1363          }
1364        }
1365      }
1366    
1367      /**
1368       * A class that represents a set of positive integer ranges. It parses 
1369       * strings of the form: "2-3,5,7-" where ranges are separated by comma and 
1370       * the lower/upper bounds are separated by dash. Either the lower or upper 
1371       * bound may be omitted meaning all values up to or over. So the string 
1372       * above means 2, 3, 5, and 7, 8, 9, ...
1373       */
1374      public static class IntegerRanges implements Iterable<Integer>{
1375        private static class Range {
1376          int start;
1377          int end;
1378        }
1379        
1380        private static class RangeNumberIterator implements Iterator<Integer> {
1381          Iterator<Range> internal;
1382          int at;
1383          int end;
1384    
1385          public RangeNumberIterator(List<Range> ranges) {
1386            if (ranges != null) {
1387              internal = ranges.iterator();
1388            }
1389            at = -1;
1390            end = -2;
1391          }
1392          
1393          @Override
1394          public boolean hasNext() {
1395            if (at <= end) {
1396              return true;
1397            } else if (internal != null){
1398              return internal.hasNext();
1399            }
1400            return false;
1401          }
1402    
1403          @Override
1404          public Integer next() {
1405            if (at <= end) {
1406              at++;
1407              return at - 1;
1408            } else if (internal != null){
1409              Range found = internal.next();
1410              if (found != null) {
1411                at = found.start;
1412                end = found.end;
1413                at++;
1414                return at - 1;
1415              }
1416            }
1417            return null;
1418          }
1419    
1420          @Override
1421          public void remove() {
1422            throw new UnsupportedOperationException();
1423          }
1424        };
1425    
1426        List<Range> ranges = new ArrayList<Range>();
1427        
1428        public IntegerRanges() {
1429        }
1430        
1431        public IntegerRanges(String newValue) {
1432          StringTokenizer itr = new StringTokenizer(newValue, ",");
1433          while (itr.hasMoreTokens()) {
1434            String rng = itr.nextToken().trim();
1435            String[] parts = rng.split("-", 3);
1436            if (parts.length < 1 || parts.length > 2) {
1437              throw new IllegalArgumentException("integer range badly formed: " + 
1438                                                 rng);
1439            }
1440            Range r = new Range();
1441            r.start = convertToInt(parts[0], 0);
1442            if (parts.length == 2) {
1443              r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1444            } else {
1445              r.end = r.start;
1446            }
1447            if (r.start > r.end) {
1448              throw new IllegalArgumentException("IntegerRange from " + r.start + 
1449                                                 " to " + r.end + " is invalid");
1450            }
1451            ranges.add(r);
1452          }
1453        }
1454    
1455        /**
1456         * Convert a string to an int treating empty strings as the default value.
1457         * @param value the string value
1458         * @param defaultValue the value for if the string is empty
1459         * @return the desired integer
1460         */
1461        private static int convertToInt(String value, int defaultValue) {
1462          String trim = value.trim();
1463          if (trim.length() == 0) {
1464            return defaultValue;
1465          }
1466          return Integer.parseInt(trim);
1467        }
1468    
1469        /**
1470         * Is the given value in the set of ranges
1471         * @param value the value to check
1472         * @return is the value in the ranges?
1473         */
1474        public boolean isIncluded(int value) {
1475          for(Range r: ranges) {
1476            if (r.start <= value && value <= r.end) {
1477              return true;
1478            }
1479          }
1480          return false;
1481        }
1482        
1483        /**
1484         * @return true if there are no values in this range, else false.
1485         */
1486        public boolean isEmpty() {
1487          return ranges == null || ranges.isEmpty();
1488        }
1489        
1490        @Override
1491        public String toString() {
1492          StringBuilder result = new StringBuilder();
1493          boolean first = true;
1494          for(Range r: ranges) {
1495            if (first) {
1496              first = false;
1497            } else {
1498              result.append(',');
1499            }
1500            result.append(r.start);
1501            result.append('-');
1502            result.append(r.end);
1503          }
1504          return result.toString();
1505        }
1506    
1507        @Override
1508        public Iterator<Integer> iterator() {
1509          return new RangeNumberIterator(ranges);
1510        }
1511        
1512      }
1513    
1514      /**
1515       * Parse the given attribute as a set of integer ranges
1516       * @param name the attribute name
1517       * @param defaultValue the default value if it is not set
1518       * @return a new set of ranges from the configured value
1519       */
1520      public IntegerRanges getRange(String name, String defaultValue) {
1521        return new IntegerRanges(get(name, defaultValue));
1522      }
1523    
1524      /** 
1525       * Get the comma delimited values of the <code>name</code> property as 
1526       * a collection of <code>String</code>s.  
1527       * If no such property is specified then empty collection is returned.
1528       * <p>
1529       * This is an optimized version of {@link #getStrings(String)}
1530       * 
1531       * @param name property name.
1532       * @return property value as a collection of <code>String</code>s. 
1533       */
1534      public Collection<String> getStringCollection(String name) {
1535        String valueString = get(name);
1536        return StringUtils.getStringCollection(valueString);
1537      }
1538    
1539      /** 
1540       * Get the comma delimited values of the <code>name</code> property as 
1541       * an array of <code>String</code>s.  
1542       * If no such property is specified then <code>null</code> is returned.
1543       * 
1544       * @param name property name.
1545       * @return property value as an array of <code>String</code>s, 
1546       *         or <code>null</code>. 
1547       */
1548      public String[] getStrings(String name) {
1549        String valueString = get(name);
1550        return StringUtils.getStrings(valueString);
1551      }
1552    
1553      /** 
1554       * Get the comma delimited values of the <code>name</code> property as 
1555       * an array of <code>String</code>s.  
1556       * If no such property is specified then default value is returned.
1557       * 
1558       * @param name property name.
1559       * @param defaultValue The default value
1560       * @return property value as an array of <code>String</code>s, 
1561       *         or default value. 
1562       */
1563      public String[] getStrings(String name, String... defaultValue) {
1564        String valueString = get(name);
1565        if (valueString == null) {
1566          return defaultValue;
1567        } else {
1568          return StringUtils.getStrings(valueString);
1569        }
1570      }
1571      
1572      /** 
1573       * Get the comma delimited values of the <code>name</code> property as 
1574       * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
1575       * If no such property is specified then empty <code>Collection</code> is returned.
1576       *
1577       * @param name property name.
1578       * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code> 
1579       */
1580      public Collection<String> getTrimmedStringCollection(String name) {
1581        String valueString = get(name);
1582        if (null == valueString) {
1583          Collection<String> empty = new ArrayList<String>();
1584          return empty;
1585        }
1586        return StringUtils.getTrimmedStringCollection(valueString);
1587      }
1588      
1589      /** 
1590       * Get the comma delimited values of the <code>name</code> property as 
1591       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1592       * If no such property is specified then an empty array is returned.
1593       * 
1594       * @param name property name.
1595       * @return property value as an array of trimmed <code>String</code>s, 
1596       *         or empty array. 
1597       */
1598      public String[] getTrimmedStrings(String name) {
1599        String valueString = get(name);
1600        return StringUtils.getTrimmedStrings(valueString);
1601      }
1602    
1603      /** 
1604       * Get the comma delimited values of the <code>name</code> property as 
1605       * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1606       * If no such property is specified then default value is returned.
1607       * 
1608       * @param name property name.
1609       * @param defaultValue The default value
1610       * @return property value as an array of trimmed <code>String</code>s, 
1611       *         or default value. 
1612       */
1613      public String[] getTrimmedStrings(String name, String... defaultValue) {
1614        String valueString = get(name);
1615        if (null == valueString) {
1616          return defaultValue;
1617        } else {
1618          return StringUtils.getTrimmedStrings(valueString);
1619        }
1620      }
1621    
1622      /** 
1623       * Set the array of string values for the <code>name</code> property as 
1624       * as comma delimited values.  
1625       * 
1626       * @param name property name.
1627       * @param values The values
1628       */
1629      public void setStrings(String name, String... values) {
1630        set(name, StringUtils.arrayToString(values));
1631      }
1632    
1633      /**
1634       * Get the socket address for <code>name</code> property as a
1635       * <code>InetSocketAddress</code>.
1636       * @param name property name.
1637       * @param defaultAddress the default value
1638       * @param defaultPort the default port
1639       * @return InetSocketAddress
1640       */
1641      public InetSocketAddress getSocketAddr(
1642          String name, String defaultAddress, int defaultPort) {
1643        final String address = get(name, defaultAddress);
1644        return NetUtils.createSocketAddr(address, defaultPort, name);
1645      }
1646    
1647      /**
1648       * Set the socket address for the <code>name</code> property as
1649       * a <code>host:port</code>.
1650       */
1651      public void setSocketAddr(String name, InetSocketAddress addr) {
1652        set(name, NetUtils.getHostPortString(addr));
1653      }
1654      
1655      /**
1656       * Set the socket address a client can use to connect for the
1657       * <code>name</code> property as a <code>host:port</code>.  The wildcard
1658       * address is replaced with the local host's address.
1659       * @param name property name.
1660       * @param addr InetSocketAddress of a listener to store in the given property
1661       * @return InetSocketAddress for clients to connect
1662       */
1663      public InetSocketAddress updateConnectAddr(String name,
1664                                                 InetSocketAddress addr) {
1665        final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
1666        setSocketAddr(name, connectAddr);
1667        return connectAddr;
1668      }
1669      
1670      /**
1671       * Load a class by name.
1672       * 
1673       * @param name the class name.
1674       * @return the class object.
1675       * @throws ClassNotFoundException if the class is not found.
1676       */
1677      public Class<?> getClassByName(String name) throws ClassNotFoundException {
1678        Class<?> ret = getClassByNameOrNull(name);
1679        if (ret == null) {
1680          throw new ClassNotFoundException("Class " + name + " not found");
1681        }
1682        return ret;
1683      }
1684      
1685      /**
1686       * Load a class by name, returning null rather than throwing an exception
1687       * if it couldn't be loaded. This is to avoid the overhead of creating
1688       * an exception.
1689       * 
1690       * @param name the class name
1691       * @return the class object, or null if it could not be found.
1692       */
1693      public Class<?> getClassByNameOrNull(String name) {
1694        Map<String, WeakReference<Class<?>>> map;
1695        
1696        synchronized (CACHE_CLASSES) {
1697          map = CACHE_CLASSES.get(classLoader);
1698          if (map == null) {
1699            map = Collections.synchronizedMap(
1700              new WeakHashMap<String, WeakReference<Class<?>>>());
1701            CACHE_CLASSES.put(classLoader, map);
1702          }
1703        }
1704    
1705        Class<?> clazz = null;
1706        WeakReference<Class<?>> ref = map.get(name); 
1707        if (ref != null) {
1708           clazz = ref.get();
1709        }
1710         
1711        if (clazz == null) {
1712          try {
1713            clazz = Class.forName(name, true, classLoader);
1714          } catch (ClassNotFoundException e) {
1715            // Leave a marker that the class isn't found
1716            map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
1717            return null;
1718          }
1719          // two putters can race here, but they'll put the same class
1720          map.put(name, new WeakReference<Class<?>>(clazz));
1721          return clazz;
1722        } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
1723          return null; // not found
1724        } else {
1725          // cache hit
1726          return clazz;
1727        }
1728      }
1729    
1730      /** 
1731       * Get the value of the <code>name</code> property
1732       * as an array of <code>Class</code>.
1733       * The value of the property specifies a list of comma separated class names.  
1734       * If no such property is specified, then <code>defaultValue</code> is 
1735       * returned.
1736       * 
1737       * @param name the property name.
1738       * @param defaultValue default value.
1739       * @return property value as a <code>Class[]</code>, 
1740       *         or <code>defaultValue</code>. 
1741       */
1742      public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
1743        String[] classnames = getTrimmedStrings(name);
1744        if (classnames == null)
1745          return defaultValue;
1746        try {
1747          Class<?>[] classes = new Class<?>[classnames.length];
1748          for(int i = 0; i < classnames.length; i++) {
1749            classes[i] = getClassByName(classnames[i]);
1750          }
1751          return classes;
1752        } catch (ClassNotFoundException e) {
1753          throw new RuntimeException(e);
1754        }
1755      }
1756    
1757      /** 
1758       * Get the value of the <code>name</code> property as a <code>Class</code>.  
1759       * If no such property is specified, then <code>defaultValue</code> is 
1760       * returned.
1761       * 
1762       * @param name the class name.
1763       * @param defaultValue default value.
1764       * @return property value as a <code>Class</code>, 
1765       *         or <code>defaultValue</code>. 
1766       */
1767      public Class<?> getClass(String name, Class<?> defaultValue) {
1768        String valueString = getTrimmed(name);
1769        if (valueString == null)
1770          return defaultValue;
1771        try {
1772          return getClassByName(valueString);
1773        } catch (ClassNotFoundException e) {
1774          throw new RuntimeException(e);
1775        }
1776      }
1777    
1778      /** 
1779       * Get the value of the <code>name</code> property as a <code>Class</code>
1780       * implementing the interface specified by <code>xface</code>.
1781       *   
1782       * If no such property is specified, then <code>defaultValue</code> is 
1783       * returned.
1784       * 
1785       * An exception is thrown if the returned class does not implement the named
1786       * interface. 
1787       * 
1788       * @param name the class name.
1789       * @param defaultValue default value.
1790       * @param xface the interface implemented by the named class.
1791       * @return property value as a <code>Class</code>, 
1792       *         or <code>defaultValue</code>.
1793       */
1794      public <U> Class<? extends U> getClass(String name, 
1795                                             Class<? extends U> defaultValue, 
1796                                             Class<U> xface) {
1797        try {
1798          Class<?> theClass = getClass(name, defaultValue);
1799          if (theClass != null && !xface.isAssignableFrom(theClass))
1800            throw new RuntimeException(theClass+" not "+xface.getName());
1801          else if (theClass != null)
1802            return theClass.asSubclass(xface);
1803          else
1804            return null;
1805        } catch (Exception e) {
1806          throw new RuntimeException(e);
1807        }
1808      }
1809    
1810      /**
1811       * Get the value of the <code>name</code> property as a <code>List</code>
1812       * of objects implementing the interface specified by <code>xface</code>.
1813       * 
1814       * An exception is thrown if any of the classes does not exist, or if it does
1815       * not implement the named interface.
1816       * 
1817       * @param name the property name.
1818       * @param xface the interface implemented by the classes named by
1819       *        <code>name</code>.
1820       * @return a <code>List</code> of objects implementing <code>xface</code>.
1821       */
1822      @SuppressWarnings("unchecked")
1823      public <U> List<U> getInstances(String name, Class<U> xface) {
1824        List<U> ret = new ArrayList<U>();
1825        Class<?>[] classes = getClasses(name);
1826        for (Class<?> cl: classes) {
1827          if (!xface.isAssignableFrom(cl)) {
1828            throw new RuntimeException(cl + " does not implement " + xface);
1829          }
1830          ret.add((U)ReflectionUtils.newInstance(cl, this));
1831        }
1832        return ret;
1833      }
1834    
1835      /** 
1836       * Set the value of the <code>name</code> property to the name of a 
1837       * <code>theClass</code> implementing the given interface <code>xface</code>.
1838       * 
1839       * An exception is thrown if <code>theClass</code> does not implement the 
1840       * interface <code>xface</code>. 
1841       * 
1842       * @param name property name.
1843       * @param theClass property value.
1844       * @param xface the interface implemented by the named class.
1845       */
1846      public void setClass(String name, Class<?> theClass, Class<?> xface) {
1847        if (!xface.isAssignableFrom(theClass))
1848          throw new RuntimeException(theClass+" not "+xface.getName());
1849        set(name, theClass.getName());
1850      }
1851    
1852      /** 
1853       * Get a local file under a directory named by <i>dirsProp</i> with
1854       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
1855       * then one is chosen based on <i>path</i>'s hash code.  If the selected
1856       * directory does not exist, an attempt is made to create it.
1857       * 
1858       * @param dirsProp directory in which to locate the file.
1859       * @param path file-path.
1860       * @return local file under the directory with the given path.
1861       */
1862      public Path getLocalPath(String dirsProp, String path)
1863        throws IOException {
1864        String[] dirs = getTrimmedStrings(dirsProp);
1865        int hashCode = path.hashCode();
1866        FileSystem fs = FileSystem.getLocal(this);
1867        for (int i = 0; i < dirs.length; i++) {  // try each local dir
1868          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1869          Path file = new Path(dirs[index], path);
1870          Path dir = file.getParent();
1871          if (fs.mkdirs(dir) || fs.exists(dir)) {
1872            return file;
1873          }
1874        }
1875        LOG.warn("Could not make " + path + 
1876                 " in local directories from " + dirsProp);
1877        for(int i=0; i < dirs.length; i++) {
1878          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1879          LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
1880        }
1881        throw new IOException("No valid local directories in property: "+dirsProp);
1882      }
1883    
1884      /** 
1885       * Get a local file name under a directory named in <i>dirsProp</i> with
1886       * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
1887       * then one is chosen based on <i>path</i>'s hash code.  If the selected
1888       * directory does not exist, an attempt is made to create it.
1889       * 
1890       * @param dirsProp directory in which to locate the file.
1891       * @param path file-path.
1892       * @return local file under the directory with the given path.
1893       */
1894      public File getFile(String dirsProp, String path)
1895        throws IOException {
1896        String[] dirs = getTrimmedStrings(dirsProp);
1897        int hashCode = path.hashCode();
1898        for (int i = 0; i < dirs.length; i++) {  // try each local dir
1899          int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1900          File file = new File(dirs[index], path);
1901          File dir = file.getParentFile();
1902          if (dir.exists() || dir.mkdirs()) {
1903            return file;
1904          }
1905        }
1906        throw new IOException("No valid local directories in property: "+dirsProp);
1907      }
1908    
1909      /** 
1910       * Get the {@link URL} for the named resource.
1911       * 
1912       * @param name resource name.
1913       * @return the url for the named resource.
1914       */
1915      public URL getResource(String name) {
1916        return classLoader.getResource(name);
1917      }
1918      
1919      /** 
1920       * Get an input stream attached to the configuration resource with the
1921       * given <code>name</code>.
1922       * 
1923       * @param name configuration resource name.
1924       * @return an input stream attached to the resource.
1925       */
1926      public InputStream getConfResourceAsInputStream(String name) {
1927        try {
1928          URL url= getResource(name);
1929    
1930          if (url == null) {
1931            LOG.info(name + " not found");
1932            return null;
1933          } else {
1934            LOG.info("found resource " + name + " at " + url);
1935          }
1936    
1937          return url.openStream();
1938        } catch (Exception e) {
1939          return null;
1940        }
1941      }
1942    
1943      /** 
1944       * Get a {@link Reader} attached to the configuration resource with the
1945       * given <code>name</code>.
1946       * 
1947       * @param name configuration resource name.
1948       * @return a reader attached to the resource.
1949       */
1950      public Reader getConfResourceAsReader(String name) {
1951        try {
1952          URL url= getResource(name);
1953    
1954          if (url == null) {
1955            LOG.info(name + " not found");
1956            return null;
1957          } else {
1958            LOG.info("found resource " + name + " at " + url);
1959          }
1960    
1961          return new InputStreamReader(url.openStream());
1962        } catch (Exception e) {
1963          return null;
1964        }
1965      }
1966    
1967      protected synchronized Properties getProps() {
1968        if (properties == null) {
1969          properties = new Properties();
1970          HashMap<String, String[]> backup = 
1971            new HashMap<String, String[]>(updatingResource);
1972          loadResources(properties, resources, quietmode);
1973          if (overlay!= null) {
1974            properties.putAll(overlay);
1975            for (Map.Entry<Object,Object> item: overlay.entrySet()) {
1976              String key = (String)item.getKey();
1977              updatingResource.put(key, backup.get(key));
1978            }
1979          }
1980        }
1981        return properties;
1982      }
1983    
1984      /**
1985       * Return the number of keys in the configuration.
1986       *
1987       * @return number of keys in the configuration.
1988       */
1989      public int size() {
1990        return getProps().size();
1991      }
1992    
1993      /**
1994       * Clears all keys from the configuration.
1995       */
1996      public void clear() {
1997        getProps().clear();
1998        getOverlay().clear();
1999      }
2000    
2001      /**
2002       * Get an {@link Iterator} to go through the list of <code>String</code> 
2003       * key-value pairs in the configuration.
2004       * 
2005       * @return an iterator over the entries.
2006       */
2007      public Iterator<Map.Entry<String, String>> iterator() {
2008        // Get a copy of just the string to string pairs. After the old object
2009        // methods that allow non-strings to be put into configurations are removed,
2010        // we could replace properties with a Map<String,String> and get rid of this
2011        // code.
2012        Map<String,String> result = new HashMap<String,String>();
2013        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2014          if (item.getKey() instanceof String && 
2015              item.getValue() instanceof String) {
2016            result.put((String) item.getKey(), (String) item.getValue());
2017          }
2018        }
2019        return result.entrySet().iterator();
2020      }
2021    
2022      private Document parse(DocumentBuilder builder, URL url)
2023          throws IOException, SAXException {
2024        if (!quietmode) {
2025          LOG.info("parsing URL " + url);
2026        }
2027        if (url == null) {
2028          return null;
2029        }
2030        return parse(builder, url.openStream(), url.toString());
2031      }
2032    
2033      private Document parse(DocumentBuilder builder, InputStream is,
2034          String systemId) throws IOException, SAXException {
2035        if (!quietmode) {
2036          LOG.info("parsing input stream " + is);
2037        }
2038        if (is == null) {
2039          return null;
2040        }
2041        try {
2042          return (systemId == null) ? builder.parse(is) : builder.parse(is,
2043              systemId);
2044        } finally {
2045          is.close();
2046        }
2047      }
2048    
2049      private void loadResources(Properties properties,
2050                                 ArrayList<Resource> resources,
2051                                 boolean quiet) {
2052        if(loadDefaults) {
2053          for (String resource : defaultResources) {
2054            loadResource(properties, new Resource(resource), quiet);
2055          }
2056        
2057          //support the hadoop-site.xml as a deprecated case
2058          if(getResource("hadoop-site.xml")!=null) {
2059            loadResource(properties, new Resource("hadoop-site.xml"), quiet);
2060          }
2061        }
2062        
2063        for (int i = 0; i < resources.size(); i++) {
2064          Resource ret = loadResource(properties, resources.get(i), quiet);
2065          if (ret != null) {
2066            resources.set(i, ret);
2067          }
2068        }
2069      }
2070      
2071      private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
2072        String name = UNKNOWN_RESOURCE;
2073        try {
2074          Object resource = wrapper.getResource();
2075          name = wrapper.getName();
2076          
2077          DocumentBuilderFactory docBuilderFactory 
2078            = DocumentBuilderFactory.newInstance();
2079          //ignore all comments inside the xml file
2080          docBuilderFactory.setIgnoringComments(true);
2081    
2082          //allow includes in the xml file
2083          docBuilderFactory.setNamespaceAware(true);
2084          try {
2085              docBuilderFactory.setXIncludeAware(true);
2086          } catch (UnsupportedOperationException e) {
2087            LOG.error("Failed to set setXIncludeAware(true) for parser "
2088                    + docBuilderFactory
2089                    + ":" + e,
2090                    e);
2091          }
2092          DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
2093          Document doc = null;
2094          Element root = null;
2095          boolean returnCachedProperties = false;
2096          
2097          if (resource instanceof URL) {                  // an URL resource
2098            doc = parse(builder, (URL)resource);
2099          } else if (resource instanceof String) {        // a CLASSPATH resource
2100            URL url = getResource((String)resource);
2101            doc = parse(builder, url);
2102          } else if (resource instanceof Path) {          // a file resource
2103            // Can't use FileSystem API or we get an infinite loop
2104            // since FileSystem uses Configuration API.  Use java.io.File instead.
2105            File file = new File(((Path)resource).toUri().getPath())
2106              .getAbsoluteFile();
2107            if (file.exists()) {
2108              if (!quiet) {
2109                LOG.info("parsing File " + file);
2110              }
2111              doc = parse(builder, new BufferedInputStream(
2112                  new FileInputStream(file)), ((Path)resource).toString());
2113            }
2114          } else if (resource instanceof InputStream) {
2115            doc = parse(builder, (InputStream) resource, null);
2116            returnCachedProperties = true;
2117          } else if (resource instanceof Properties) {
2118            overlay(properties, (Properties)resource);
2119          } else if (resource instanceof Element) {
2120            root = (Element)resource;
2121          }
2122    
2123          if (doc == null && root == null) {
2124            if (quiet)
2125              return null;
2126            throw new RuntimeException(resource + " not found");
2127          }
2128    
2129          if (root == null) {
2130            root = doc.getDocumentElement();
2131          }
2132          Properties toAddTo = properties;
2133          if(returnCachedProperties) {
2134            toAddTo = new Properties();
2135          }
2136          if (!"configuration".equals(root.getTagName()))
2137            LOG.fatal("bad conf file: top-level element not <configuration>");
2138          NodeList props = root.getChildNodes();
2139          DeprecationContext deprecations = deprecationContext.get();
2140          for (int i = 0; i < props.getLength(); i++) {
2141            Node propNode = props.item(i);
2142            if (!(propNode instanceof Element))
2143              continue;
2144            Element prop = (Element)propNode;
2145            if ("configuration".equals(prop.getTagName())) {
2146              loadResource(toAddTo, new Resource(prop, name), quiet);
2147              continue;
2148            }
2149            if (!"property".equals(prop.getTagName()))
2150              LOG.warn("bad conf file: element not <property>");
2151            NodeList fields = prop.getChildNodes();
2152            String attr = null;
2153            String value = null;
2154            boolean finalParameter = false;
2155            LinkedList<String> source = new LinkedList<String>();
2156            for (int j = 0; j < fields.getLength(); j++) {
2157              Node fieldNode = fields.item(j);
2158              if (!(fieldNode instanceof Element))
2159                continue;
2160              Element field = (Element)fieldNode;
2161              if ("name".equals(field.getTagName()) && field.hasChildNodes())
2162                attr = StringInterner.weakIntern(
2163                    ((Text)field.getFirstChild()).getData().trim());
2164              if ("value".equals(field.getTagName()) && field.hasChildNodes())
2165                value = StringInterner.weakIntern(
2166                    ((Text)field.getFirstChild()).getData());
2167              if ("final".equals(field.getTagName()) && field.hasChildNodes())
2168                finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
2169              if ("source".equals(field.getTagName()) && field.hasChildNodes())
2170                source.add(StringInterner.weakIntern(
2171                    ((Text)field.getFirstChild()).getData()));
2172            }
2173            source.add(name);
2174            
2175            // Ignore this parameter if it has already been marked as 'final'
2176            if (attr != null) {
2177              if (deprecations.getDeprecatedKeyMap().containsKey(attr)) {
2178                DeprecatedKeyInfo keyInfo =
2179                    deprecations.getDeprecatedKeyMap().get(attr);
2180                keyInfo.clearAccessed();
2181                for (String key:keyInfo.newKeys) {
2182                  // update new keys with deprecated key's value 
2183                  loadProperty(toAddTo, name, key, value, finalParameter, 
2184                      source.toArray(new String[source.size()]));
2185                }
2186              }
2187              else {
2188                loadProperty(toAddTo, name, attr, value, finalParameter, 
2189                    source.toArray(new String[source.size()]));
2190              }
2191            }
2192          }
2193          
2194          if (returnCachedProperties) {
2195            overlay(properties, toAddTo);
2196            return new Resource(toAddTo, name);
2197          }
2198          return null;
2199        } catch (IOException e) {
2200          LOG.fatal("error parsing conf " + name, e);
2201          throw new RuntimeException(e);
2202        } catch (DOMException e) {
2203          LOG.fatal("error parsing conf " + name, e);
2204          throw new RuntimeException(e);
2205        } catch (SAXException e) {
2206          LOG.fatal("error parsing conf " + name, e);
2207          throw new RuntimeException(e);
2208        } catch (ParserConfigurationException e) {
2209          LOG.fatal("error parsing conf " + name , e);
2210          throw new RuntimeException(e);
2211        }
2212      }
2213    
2214      private void overlay(Properties to, Properties from) {
2215        for (Entry<Object, Object> entry: from.entrySet()) {
2216          to.put(entry.getKey(), entry.getValue());
2217        }
2218      }
2219      
2220      private void loadProperty(Properties properties, String name, String attr,
2221          String value, boolean finalParameter, String[] source) {
2222        if (value != null) {
2223          if (!finalParameters.contains(attr)) {
2224            properties.setProperty(attr, value);
2225            updatingResource.put(attr, source);
2226          } else if (!value.equals(properties.getProperty(attr))) {
2227            LOG.warn(name+":an attempt to override final parameter: "+attr
2228                +";  Ignoring.");
2229          }
2230        }
2231        if (finalParameter) {
2232          finalParameters.add(attr);
2233        }
2234      }
2235    
2236      /** 
2237       * Write out the non-default properties in this configuration to the given
2238       * {@link OutputStream}.
2239       * 
2240       * @param out the output stream to write to.
2241       */
2242      public void writeXml(OutputStream out) throws IOException {
2243        writeXml(new OutputStreamWriter(out));
2244      }
2245    
2246      /** 
2247       * Write out the non-default properties in this configuration to the given
2248       * {@link Writer}.
2249       * 
2250       * @param out the writer to write to.
2251       */
2252      public void writeXml(Writer out) throws IOException {
2253        Document doc = asXmlDocument();
2254    
2255        try {
2256          DOMSource source = new DOMSource(doc);
2257          StreamResult result = new StreamResult(out);
2258          TransformerFactory transFactory = TransformerFactory.newInstance();
2259          Transformer transformer = transFactory.newTransformer();
2260    
2261          // Important to not hold Configuration log while writing result, since
2262          // 'out' may be an HDFS stream which needs to lock this configuration
2263          // from another thread.
2264          transformer.transform(source, result);
2265        } catch (TransformerException te) {
2266          throw new IOException(te);
2267        }
2268      }
2269    
2270      /**
2271       * Return the XML DOM corresponding to this Configuration.
2272       */
2273      private synchronized Document asXmlDocument() throws IOException {
2274        Document doc;
2275        try {
2276          doc =
2277            DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
2278        } catch (ParserConfigurationException pe) {
2279          throw new IOException(pe);
2280        }
2281        Element conf = doc.createElement("configuration");
2282        doc.appendChild(conf);
2283        conf.appendChild(doc.createTextNode("\n"));
2284        handleDeprecation(); //ensure properties is set and deprecation is handled
2285        for (Enumeration e = properties.keys(); e.hasMoreElements();) {
2286          String name = (String)e.nextElement();
2287          Object object = properties.get(name);
2288          String value = null;
2289          if (object instanceof String) {
2290            value = (String) object;
2291          }else {
2292            continue;
2293          }
2294          Element propNode = doc.createElement("property");
2295          conf.appendChild(propNode);
2296    
2297          Element nameNode = doc.createElement("name");
2298          nameNode.appendChild(doc.createTextNode(name));
2299          propNode.appendChild(nameNode);
2300    
2301          Element valueNode = doc.createElement("value");
2302          valueNode.appendChild(doc.createTextNode(value));
2303          propNode.appendChild(valueNode);
2304    
2305          if (updatingResource != null) {
2306            String[] sources = updatingResource.get(name);
2307            if(sources != null) {
2308              for(String s : sources) {
2309                Element sourceNode = doc.createElement("source");
2310                sourceNode.appendChild(doc.createTextNode(s));
2311                propNode.appendChild(sourceNode);
2312              }
2313            }
2314          }
2315          
2316          conf.appendChild(doc.createTextNode("\n"));
2317        }
2318        return doc;
2319      }
2320    
2321      /**
2322       *  Writes out all the parameters and their properties (final and resource) to
2323       *  the given {@link Writer}
2324       *  The format of the output would be 
2325       *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
2326       *  key2.isFinal,key2.resource}... ] } 
2327       *  It does not output the parameters of the configuration object which is 
2328       *  loaded from an input stream.
2329       * @param out the Writer to write to
2330       * @throws IOException
2331       */
2332      public static void dumpConfiguration(Configuration config,
2333          Writer out) throws IOException {
2334        JsonFactory dumpFactory = new JsonFactory();
2335        JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
2336        dumpGenerator.writeStartObject();
2337        dumpGenerator.writeFieldName("properties");
2338        dumpGenerator.writeStartArray();
2339        dumpGenerator.flush();
2340        synchronized (config) {
2341          for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
2342            dumpGenerator.writeStartObject();
2343            dumpGenerator.writeStringField("key", (String) item.getKey());
2344            dumpGenerator.writeStringField("value", 
2345                                           config.get((String) item.getKey()));
2346            dumpGenerator.writeBooleanField("isFinal",
2347                                            config.finalParameters.contains(item.getKey()));
2348            String[] resources = config.updatingResource.get(item.getKey());
2349            String resource = UNKNOWN_RESOURCE;
2350            if(resources != null && resources.length > 0) {
2351              resource = resources[0];
2352            }
2353            dumpGenerator.writeStringField("resource", resource);
2354            dumpGenerator.writeEndObject();
2355          }
2356        }
2357        dumpGenerator.writeEndArray();
2358        dumpGenerator.writeEndObject();
2359        dumpGenerator.flush();
2360      }
2361      
2362      /**
2363       * Get the {@link ClassLoader} for this job.
2364       * 
2365       * @return the correct class loader.
2366       */
2367      public ClassLoader getClassLoader() {
2368        return classLoader;
2369      }
2370      
2371      /**
2372       * Set the class loader that will be used to load the various objects.
2373       * 
2374       * @param classLoader the new class loader.
2375       */
2376      public void setClassLoader(ClassLoader classLoader) {
2377        this.classLoader = classLoader;
2378      }
2379      
2380      @Override
2381      public String toString() {
2382        StringBuilder sb = new StringBuilder();
2383        sb.append("Configuration: ");
2384        if(loadDefaults) {
2385          toString(defaultResources, sb);
2386          if(resources.size()>0) {
2387            sb.append(", ");
2388          }
2389        }
2390        toString(resources, sb);
2391        return sb.toString();
2392      }
2393      
2394      private <T> void toString(List<T> resources, StringBuilder sb) {
2395        ListIterator<T> i = resources.listIterator();
2396        while (i.hasNext()) {
2397          if (i.nextIndex() != 0) {
2398            sb.append(", ");
2399          }
2400          sb.append(i.next());
2401        }
2402      }
2403    
2404      /** 
2405       * Set the quietness-mode. 
2406       * 
2407       * In the quiet-mode, error and informational messages might not be logged.
2408       * 
2409       * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2410       *              to turn it off.
2411       */
2412      public synchronized void setQuietMode(boolean quietmode) {
2413        this.quietmode = quietmode;
2414      }
2415    
2416      synchronized boolean getQuietMode() {
2417        return this.quietmode;
2418      }
2419      
2420      /** For debugging.  List non-default properties to the terminal and exit. */
2421      public static void main(String[] args) throws Exception {
2422        new Configuration().writeXml(System.out);
2423      }
2424    
2425      @Override
2426      public void readFields(DataInput in) throws IOException {
2427        clear();
2428        int size = WritableUtils.readVInt(in);
2429        for(int i=0; i < size; ++i) {
2430          String key = org.apache.hadoop.io.Text.readString(in);
2431          String value = org.apache.hadoop.io.Text.readString(in);
2432          set(key, value); 
2433          String sources[] = WritableUtils.readCompressedStringArray(in);
2434          updatingResource.put(key, sources);
2435        }
2436      }
2437    
2438      //@Override
2439      public void write(DataOutput out) throws IOException {
2440        Properties props = getProps();
2441        WritableUtils.writeVInt(out, props.size());
2442        for(Map.Entry<Object, Object> item: props.entrySet()) {
2443          org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2444          org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2445          WritableUtils.writeCompressedStringArray(out, 
2446              updatingResource.get(item.getKey()));
2447        }
2448      }
2449      
2450      /**
2451       * get keys matching the the regex 
2452       * @param regex
2453       * @return Map<String,String> with matching keys
2454       */
2455      public Map<String,String> getValByRegex(String regex) {
2456        Pattern p = Pattern.compile(regex);
2457    
2458        Map<String,String> result = new HashMap<String,String>();
2459        Matcher m;
2460    
2461        for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2462          if (item.getKey() instanceof String && 
2463              item.getValue() instanceof String) {
2464            m = p.matcher((String)item.getKey());
2465            if(m.find()) { // match
2466              result.put((String) item.getKey(), (String) item.getValue());
2467            }
2468          }
2469        }
2470        return result;
2471      }
2472    
2473      /**
2474       * A unique class which is used as a sentinel value in the caching
2475       * for getClassByName. {@link Configuration#getClassByNameOrNull(String)}
2476       */
2477      private static abstract class NegativeCacheSentinel {}
2478    
2479      public static void dumpDeprecatedKeys() {
2480        DeprecationContext deprecations = deprecationContext.get();
2481        for (Map.Entry<String, DeprecatedKeyInfo> entry :
2482            deprecations.getDeprecatedKeyMap().entrySet()) {
2483          StringBuilder newKeys = new StringBuilder();
2484          for (String newKey : entry.getValue().newKeys) {
2485            newKeys.append(newKey).append("\t");
2486          }
2487          System.out.println(entry.getKey() + "\t" + newKeys.toString());
2488        }
2489      }
2490    }