View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master.normalizer;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.hbase.HBaseIOException;
24  import org.apache.hadoop.hbase.HRegionInfo;
25  import org.apache.hadoop.hbase.RegionLoad;
26  import org.apache.hadoop.hbase.ServerName;
27  import org.apache.hadoop.hbase.TableName;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.hbase.master.MasterServices;
30  import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
31  
32  import java.util.ArrayList;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.List;
36  
37  /**
38   * Simple implementation of region normalizer.
39   *
40   * Logic in use:
41   *
42   *  <ol>
43   *  <li> get all regions of a given table
44   *  <li> get avg size S of each region (by total size of store files reported in RegionLoad)
45   *  <li> If biggest region is bigger than S * 2, it is kindly requested to split,
46   *    and normalization stops
47   *  <li> Otherwise, two smallest region R1 and its smallest neighbor R2 are kindly requested
48   *    to merge, if R1 + R1 &lt;  S, and normalization stops
49   *  <li> Otherwise, no action is performed
50   * </ol>
51   * <p>
52   * Region sizes are coarse and approximate on the order of megabytes. Additionally,
53   * "empty" regions (less than 1MB, with the previous note) are not merged away. This
54   * is by design to prevent normalization from undoing the pre-splitting of a table.
55   */
56  @InterfaceAudience.Private
57  public class SimpleRegionNormalizer implements RegionNormalizer {
58  
59    private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class);
60    private static final int MIN_REGION_COUNT = 3;
61    private MasterServices masterServices;
62  
63    /**
64     * Set the master service.
65     * @param masterServices inject instance of MasterServices
66     */
67    @Override
68    public void setMasterServices(MasterServices masterServices) {
69      this.masterServices = masterServices;
70    }
71  
72    // Comparator that gives higher priority to region Split plan
73    private Comparator<NormalizationPlan> planComparator =
74        new Comparator<NormalizationPlan>() {
75      @Override
76      public int compare(NormalizationPlan plan, NormalizationPlan plan2) {
77        if (plan instanceof SplitNormalizationPlan) {
78          return -1;
79        }
80        if (plan2 instanceof SplitNormalizationPlan) {
81          return 1;
82        }
83        return 0;
84      }
85    };
86  
87    /**
88     * Computes next most "urgent" normalization action on the table.
89     * Action may be either a split, or a merge, or no action.
90     *
91     * @param table table to normalize
92     * @return normalization plan to execute
93     */
94    @Override
95    public List<NormalizationPlan> computePlanForTable(TableName table) throws HBaseIOException {
96      if (table == null || table.isSystemTable()) {
97        LOG.debug("Normalization of system table " + table + " isn't allowed");
98        return null;
99      }
100 
101     List<NormalizationPlan> plans = new ArrayList<NormalizationPlan>();
102     List<HRegionInfo> tableRegions = masterServices.getAssignmentManager().getRegionStates().
103       getRegionsOfTable(table);
104 
105     //TODO: should we make min number of regions a config param?
106     if (tableRegions == null || tableRegions.size() < MIN_REGION_COUNT) {
107       int nrRegions = tableRegions == null ? 0 : tableRegions.size();
108       LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number"
109         + " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer");
110       return null;
111     }
112 
113     LOG.debug("Computing normalization plan for table: " + table +
114       ", number of regions: " + tableRegions.size());
115 
116     long totalSizeMb = 0;
117 
118     for (int i = 0; i < tableRegions.size(); i++) {
119       HRegionInfo hri = tableRegions.get(i);
120       long regionSize = getRegionSize(hri);
121       if (regionSize > 0) {
122         totalSizeMb += regionSize;
123       }
124     }
125 
126     double avgRegionSize = totalSizeMb / (double) tableRegions.size();
127 
128     LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb);
129     LOG.debug("Table " + table + ", average region size: " + avgRegionSize);
130 
131     int candidateIdx = 0;
132     while (candidateIdx < tableRegions.size()) {
133       HRegionInfo hri = tableRegions.get(candidateIdx);
134       long regionSize = getRegionSize(hri);
135       // if the region is > 2 times larger than average, we split it, split
136       // is more high priority normalization action than merge.
137       if (regionSize > 2 * avgRegionSize) {
138         LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size "
139             + regionSize + ", more than twice avg size, splitting");
140         plans.add(new SplitNormalizationPlan(hri, null));
141       } else {
142         if (candidateIdx == tableRegions.size()-1) {
143           break;
144         }
145         HRegionInfo hri2 = tableRegions.get(candidateIdx+1);
146         long regionSize2 = getRegionSize(hri2);
147         if (regionSize >= 0 && regionSize2 >= 0 && regionSize + regionSize2 < avgRegionSize) {
148           LOG.info("Table " + table + ", small region size: " + regionSize
149             + " plus its neighbor size: " + regionSize2
150             + ", less than the avg size " + avgRegionSize + ", merging them");
151           plans.add(new MergeNormalizationPlan(hri, hri2));
152           candidateIdx++;
153         }
154       }
155       candidateIdx++;
156     }
157     if (plans.isEmpty()) {
158       LOG.debug("No normalization needed, regions look good for table: " + table);
159       return null;
160     }
161     Collections.sort(plans, planComparator);
162     return plans;
163   }
164 
165   private long getRegionSize(HRegionInfo hri) {
166     ServerName sn = masterServices.getAssignmentManager().getRegionStates().
167       getRegionServerOfRegion(hri);
168     RegionLoad regionLoad = masterServices.getServerManager().getLoad(sn).
169       getRegionsLoad().get(hri.getRegionName());
170     if (regionLoad == null) {
171       LOG.debug(hri.getRegionNameAsString() + " was not found in RegionsLoad");
172       return -1;
173     }
174     return regionLoad.getStorefileSizeMB();
175   }
176 }