View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.List;
31  import java.util.Random;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HRegionLocation;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
41  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
42  import org.apache.hadoop.hbase.testclassification.LargeTests;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.Pair;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  @Category(LargeTests.class)
53  public class TestFromClientSide3 {
54    final Log LOG = LogFactory.getLog(getClass());
55    private final static HBaseTestingUtility TEST_UTIL
56      = new HBaseTestingUtility();
57    private static byte[] FAMILY = Bytes.toBytes("testFamily");
58    private static Random random = new Random();
59    private static int SLAVES = 3;
60    private static byte [] ROW = Bytes.toBytes("testRow");
61    private static final byte[] ANOTHERROW = Bytes.toBytes("anotherrow");
62    private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
63    private static byte [] VALUE = Bytes.toBytes("testValue");
64    private final static byte[] COL_QUAL = Bytes.toBytes("f1");
65    private final static byte[] VAL_BYTES = Bytes.toBytes("v1");
66    private final static byte[] ROW_BYTES = Bytes.toBytes("r1");
67  
68    /**
69     * @throws java.lang.Exception
70     */
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      TEST_UTIL.getConfiguration().setBoolean(
74          "hbase.online.schema.update.enable", true);
75      TEST_UTIL.startMiniCluster(SLAVES);
76    }
77  
78    /**
79     * @throws java.lang.Exception
80     */
81    @AfterClass
82    public static void tearDownAfterClass() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    /**
87     * @throws java.lang.Exception
88     */
89    @Before
90    public void setUp() throws Exception {
91      // Nothing to do.
92    }
93  
94    /**
95     * @throws java.lang.Exception
96     */
97    @After
98    public void tearDown() throws Exception {
99      for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) {
100       LOG.info("Tear down, remove table=" + htd.getTableName());
101       TEST_UTIL.deleteTable(htd.getTableName());
102     }
103   }
104 
105   private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts)
106       throws Exception {
107     Put put = new Put(row);
108     for (int i = 0; i < nPuts; i++) {
109       byte[] qualifier = Bytes.toBytes(random.nextInt());
110       byte[] value = Bytes.toBytes(random.nextInt());
111       put.add(family, qualifier, value);
112     }
113     table.put(put);
114   }
115 
116   private void performMultiplePutAndFlush(HBaseAdmin admin, HTable table,
117       byte[] row, byte[] family, int nFlushes, int nPuts)
118   throws Exception {
119 
120     // connection needed for poll-wait
121     HRegionLocation loc = table.getRegionLocation(row, true);
122     AdminProtos.AdminService.BlockingInterface server =
123       admin.getConnection().getAdmin(loc.getServerName());
124     byte[] regName = loc.getRegionInfo().getRegionName();
125 
126     for (int i = 0; i < nFlushes; i++) {
127       randomCFPuts(table, row, family, nPuts);
128       List<String> sf = ProtobufUtil.getStoreFiles(server, regName, FAMILY);
129       int sfCount = sf.size();
130 
131       // TODO: replace this api with a synchronous flush after HBASE-2949
132       admin.flush(table.getTableName());
133 
134       // synchronously poll wait for a new storefile to appear (flush happened)
135       while (ProtobufUtil.getStoreFiles(
136           server, regName, FAMILY).size() == sfCount) {
137         Thread.sleep(40);
138       }
139     }
140   }
141 
142   // override the config settings at the CF level and ensure priority
143   @Test(timeout = 60000)
144   public void testAdvancedConfigOverride() throws Exception {
145     /*
146      * Overall idea: (1) create 3 store files and issue a compaction. config's
147      * compaction.min == 3, so should work. (2) Increase the compaction.min
148      * toggle in the HTD to 5 and modify table. If we use the HTD value instead
149      * of the default config value, adding 3 files and issuing a compaction
150      * SHOULD NOT work (3) Decrease the compaction.min toggle in the HCD to 2
151      * and modify table. The CF schema should override the Table schema and now
152      * cause a minor compaction.
153      */
154     TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 3);
155 
156     String tableName = "testAdvancedConfigOverride";
157     TableName TABLE = TableName.valueOf(tableName);
158     HTable hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
159     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
160     ClusterConnection connection = (ClusterConnection)TEST_UTIL.getConnection();
161 
162     // Create 3 store files.
163     byte[] row = Bytes.toBytes(random.nextInt());
164     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 100);
165 
166     // Verify we have multiple store files.
167     HRegionLocation loc = hTable.getRegionLocation(row, true);
168     byte[] regionName = loc.getRegionInfo().getRegionName();
169     AdminProtos.AdminService.BlockingInterface server =
170       connection.getAdmin(loc.getServerName());
171     assertTrue(ProtobufUtil.getStoreFiles(
172       server, regionName, FAMILY).size() > 1);
173 
174     // Issue a compaction request
175     admin.compact(TABLE.getName());
176 
177     // poll wait for the compactions to happen
178     for (int i = 0; i < 10 * 1000 / 40; ++i) {
179       // The number of store files after compaction should be lesser.
180       loc = hTable.getRegionLocation(row, true);
181       if (!loc.getRegionInfo().isOffline()) {
182         regionName = loc.getRegionInfo().getRegionName();
183         server = connection.getAdmin(loc.getServerName());
184         if (ProtobufUtil.getStoreFiles(
185             server, regionName, FAMILY).size() <= 1) {
186           break;
187         }
188       }
189       Thread.sleep(40);
190     }
191     // verify the compactions took place and that we didn't just time out
192     assertTrue(ProtobufUtil.getStoreFiles(
193       server, regionName, FAMILY).size() <= 1);
194 
195     // change the compaction.min config option for this table to 5
196     LOG.info("hbase.hstore.compaction.min should now be 5");
197     HTableDescriptor htd = new HTableDescriptor(hTable.getTableDescriptor());
198     htd.setValue("hbase.hstore.compaction.min", String.valueOf(5));
199     admin.modifyTable(TABLE, htd);
200     Pair<Integer, Integer> st;
201     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
202       LOG.debug(st.getFirst() + " regions left to update");
203       Thread.sleep(40);
204     }
205     LOG.info("alter status finished");
206 
207     // Create 3 more store files.
208     performMultiplePutAndFlush(admin, hTable, row, FAMILY, 3, 10);
209 
210     // Issue a compaction request
211     admin.compact(TABLE.getName());
212 
213     // This time, the compaction request should not happen
214     Thread.sleep(10 * 1000);
215     loc = hTable.getRegionLocation(row, true);
216     regionName = loc.getRegionInfo().getRegionName();
217     server = connection.getAdmin(loc.getServerName());
218     int sfCount = ProtobufUtil.getStoreFiles(
219       server, regionName, FAMILY).size();
220     assertTrue(sfCount > 1);
221 
222     // change an individual CF's config option to 2 & online schema update
223     LOG.info("hbase.hstore.compaction.min should now be 2");
224     HColumnDescriptor hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
225     hcd.setValue("hbase.hstore.compaction.min", String.valueOf(2));
226     htd.modifyFamily(hcd);
227     admin.modifyTable(TABLE, htd);
228     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
229       LOG.debug(st.getFirst() + " regions left to update");
230       Thread.sleep(40);
231     }
232     LOG.info("alter status finished");
233 
234     // Issue a compaction request
235     admin.compact(TABLE.getName());
236 
237     // poll wait for the compactions to happen
238     for (int i = 0; i < 10 * 1000 / 40; ++i) {
239       loc = hTable.getRegionLocation(row, true);
240       regionName = loc.getRegionInfo().getRegionName();
241       try {
242         server = connection.getAdmin(loc.getServerName());
243         if (ProtobufUtil.getStoreFiles(
244             server, regionName, FAMILY).size() < sfCount) {
245           break;
246         }
247       } catch (Exception e) {
248         LOG.debug("Waiting for region to come online: " + regionName);
249       }
250       Thread.sleep(40);
251     }
252     // verify the compaction took place and that we didn't just time out
253     assertTrue(ProtobufUtil.getStoreFiles(
254       server, regionName, FAMILY).size() < sfCount);
255 
256     // Finally, ensure that we can remove a custom config value after we made it
257     LOG.info("Removing CF config value");
258     LOG.info("hbase.hstore.compaction.min should now be 5");
259     hcd = new HColumnDescriptor(htd.getFamily(FAMILY));
260     hcd.setValue("hbase.hstore.compaction.min", null);
261     htd.modifyFamily(hcd);
262     admin.modifyTable(TABLE, htd);
263     while (null != (st = admin.getAlterStatus(TABLE)) && st.getFirst() > 0) {
264       LOG.debug(st.getFirst() + " regions left to update");
265       Thread.sleep(40);
266     }
267     LOG.info("alter status finished");
268     assertNull(hTable.getTableDescriptor().getFamily(FAMILY).getValue(
269         "hbase.hstore.compaction.min"));
270   }
271 
272   @Test
273   public void testHTableBatchWithEmptyPut() throws Exception {
274     Table table = TEST_UTIL.createTable(
275       Bytes.toBytes("testHTableBatchWithEmptyPut"), new byte[][] { FAMILY });
276     try {
277       List actions = (List) new ArrayList();
278       Object[] results = new Object[2];
279       // create an empty Put
280       Put put1 = new Put(ROW);
281       actions.add(put1);
282 
283       Put put2 = new Put(ANOTHERROW);
284       put2.add(FAMILY, QUALIFIER, VALUE);
285       actions.add(put2);
286 
287       table.batch(actions, results);
288       fail("Empty Put should have failed the batch call");
289     } catch (IllegalArgumentException iae) {
290 
291     } finally {
292       table.close();
293     }
294   }
295 
296   @Test
297   public void testHTableExistsMethodSingleRegionSingleGet() throws Exception {
298 
299     // Test with a single region table.
300 
301     Table table = TEST_UTIL.createTable(
302       Bytes.toBytes("testHTableExistsMethodSingleRegionSingleGet"), new byte[][] { FAMILY });
303 
304     Put put = new Put(ROW);
305     put.add(FAMILY, QUALIFIER, VALUE);
306 
307     Get get = new Get(ROW);
308 
309     boolean exist = table.exists(get);
310     assertEquals(exist, false);
311 
312     table.put(put);
313 
314     exist = table.exists(get);
315     assertEquals(exist, true);
316   }
317 
318   public void testHTableExistsMethodSingleRegionMultipleGets() throws Exception {
319 
320     HTable table = TEST_UTIL.createTable(
321       Bytes.toBytes("testHTableExistsMethodSingleRegionMultipleGets"), new byte[][] { FAMILY });
322 
323     Put put = new Put(ROW);
324     put.add(FAMILY, QUALIFIER, VALUE);
325     table.put(put);
326 
327     List<Get> gets = new ArrayList<Get>();
328     gets.add(new Get(ROW));
329     gets.add(null);
330     gets.add(new Get(ANOTHERROW));
331 
332     Boolean[] results = table.exists(gets);
333     assertEquals(results[0], true);
334     assertEquals(results[1], false);
335     assertEquals(results[2], false);
336   }
337 
338   @Test
339   public void testHTableExistsBeforeGet() throws Exception {
340     Table table = TEST_UTIL.createTable(
341       Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY });
342     try {
343       Put put = new Put(ROW);
344       put.add(FAMILY, QUALIFIER, VALUE);
345       table.put(put);
346 
347       Get get = new Get(ROW);
348 
349       boolean exist = table.exists(get);
350       assertEquals(true, exist);
351 
352       Result result = table.get(get);
353       assertEquals(false, result.isEmpty());
354       assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER)));
355     } finally {
356       table.close();
357     }
358   }
359 
360   @Test
361   public void testHTableExistsAllBeforeGet() throws Exception {
362     final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2"));
363     Table table = TEST_UTIL.createTable(
364       Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY });
365     try {
366       Put put = new Put(ROW);
367       put.add(FAMILY, QUALIFIER, VALUE);
368       table.put(put);
369       put = new Put(ROW2);
370       put.add(FAMILY, QUALIFIER, VALUE);
371       table.put(put);
372 
373       Get get = new Get(ROW);
374       Get get2 = new Get(ROW2);
375       ArrayList<Get> getList = new ArrayList(2);
376       getList.add(get);
377       getList.add(get2);
378 
379       boolean[] exists = table.existsAll(getList);
380       assertEquals(true, exists[0]);
381       assertEquals(true, exists[1]);
382 
383       Result[] result = table.get(getList);
384       assertEquals(false, result[0].isEmpty());
385       assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER)));
386       assertEquals(false, result[1].isEmpty());
387       assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER)));
388     } finally {
389       table.close();
390     }
391   }
392 
393   @Test
394   public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception {
395 
396     Table table = TEST_UTIL.createTable(
397       TableName.valueOf("testHTableExistsMethodMultipleRegionsSingleGet"), new byte[][] { FAMILY },
398       1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
399     Put put = new Put(ROW);
400     put.add(FAMILY, QUALIFIER, VALUE);
401 
402     Get get = new Get(ROW);
403 
404     boolean exist = table.exists(get);
405     assertEquals(exist, false);
406 
407     table.put(put);
408 
409     exist = table.exists(get);
410     assertEquals(exist, true);
411   }
412 
413   @Test
414   public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception {
415     HTable table = TEST_UTIL.createTable(
416       TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"),
417       new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255);
418     Put put = new Put(ROW);
419     put.add(FAMILY, QUALIFIER, VALUE);
420     table.put (put);
421 
422     List<Get> gets = new ArrayList<Get>();
423     gets.add(new Get(ANOTHERROW));
424     gets.add(new Get(Bytes.add(ROW, new byte[] { 0x00 })));
425     gets.add(new Get(ROW));
426     gets.add(new Get(Bytes.add(ANOTHERROW, new byte[] { 0x00 })));
427 
428     LOG.info("Calling exists");
429     Boolean[] results = table.exists(gets);
430     assertEquals(results[0], false);
431     assertEquals(results[1], false);
432     assertEquals(results[2], true);
433     assertEquals(results[3], false);
434 
435     // Test with the first region.
436     put = new Put(new byte[] { 0x00 });
437     put.add(FAMILY, QUALIFIER, VALUE);
438     table.put(put);
439 
440     gets = new ArrayList<Get>();
441     gets.add(new Get(new byte[] { 0x00 }));
442     gets.add(new Get(new byte[] { 0x00, 0x00 }));
443     results = table.exists(gets);
444     assertEquals(results[0], true);
445     assertEquals(results[1], false);
446 
447     // Test with the last region
448     put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
449     put.add(FAMILY, QUALIFIER, VALUE);
450     table.put(put);
451 
452     gets = new ArrayList<Get>();
453     gets.add(new Get(new byte[] { (byte) 0xff }));
454     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
455     gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
456     results = table.exists(gets);
457     assertEquals(results[0], false);
458     assertEquals(results[1], true);
459     assertEquals(results[2], false);
460   }
461 
462   @Test
463   public void testGetEmptyRow() throws Exception {
464     //Create a table and put in 1 row
465     Admin admin = TEST_UTIL.getHBaseAdmin();
466     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("test")));
467     desc.addFamily(new HColumnDescriptor(FAMILY));
468     admin.createTable(desc);
469     Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
470 
471     Put put = new Put(ROW_BYTES);
472     put.add(FAMILY, COL_QUAL, VAL_BYTES);
473     table.put(put);
474 
475     //Try getting the row with an empty row key
476     Result res = null;
477     try {
478       res = table.get(new Get(new byte[0]));
479       fail();
480     } catch (IllegalArgumentException e) {
481       // Expected.
482     }
483     assertTrue(res == null);
484     res = table.get(new Get(Bytes.toBytes("r1-not-exist")));
485     assertTrue(res.isEmpty() == true);
486     res = table.get(new Get(ROW_BYTES));
487     assertTrue(Arrays.equals(res.getValue(FAMILY, COL_QUAL), VAL_BYTES));
488     table.close();
489   }
490 }