View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotNull;
25  import static org.junit.Assert.assertNull;
26  import static org.junit.Assert.assertSame;
27  import static org.junit.Assert.assertTrue;
28  import static org.junit.Assert.fail;
29  
30  import java.io.IOException;
31  import java.lang.reflect.Method;
32  import java.util.ArrayList;
33  import java.util.Arrays;
34  import java.util.Collections;
35  import java.util.HashSet;
36  import java.util.Iterator;
37  import java.util.List;
38  import java.util.Map;
39  import java.util.NavigableMap;
40  import java.util.UUID;
41  import java.util.concurrent.Callable;
42  import java.util.concurrent.ExecutorService;
43  import java.util.concurrent.Executors;
44  import java.util.concurrent.atomic.AtomicReference;
45  
46  import org.apache.hadoop.mapreduce.Cluster;
47  import org.apache.log4j.Level;
48  import org.apache.commons.lang.ArrayUtils;
49  import org.apache.commons.logging.Log;
50  import org.apache.commons.logging.LogFactory;
51  import org.apache.hadoop.conf.Configuration;
52  import org.apache.hadoop.hbase.Abortable;
53  import org.apache.hadoop.hbase.Cell;
54  import org.apache.hadoop.hbase.CellUtil;
55  import org.apache.hadoop.hbase.HBaseTestingUtility;
56  import org.apache.hadoop.hbase.HColumnDescriptor;
57  import org.apache.hadoop.hbase.HConstants;
58  import org.apache.hadoop.hbase.HRegionInfo;
59  import org.apache.hadoop.hbase.HRegionLocation;
60  import org.apache.hadoop.hbase.HTableDescriptor;
61  import org.apache.hadoop.hbase.KeyValue;
62  import org.apache.hadoop.hbase.MiniHBaseCluster;
63  import org.apache.hadoop.hbase.RegionLocations;
64  import org.apache.hadoop.hbase.ServerName;
65  import org.apache.hadoop.hbase.TableName;
66  import org.apache.hadoop.hbase.Waiter;
67  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
68  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
69  import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
70  import org.apache.hadoop.hbase.filter.BinaryComparator;
71  import org.apache.hadoop.hbase.filter.CompareFilter;
72  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
73  import org.apache.hadoop.hbase.filter.Filter;
74  import org.apache.hadoop.hbase.filter.FilterList;
75  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
76  import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
77  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
78  import org.apache.hadoop.hbase.filter.LongComparator;
79  import org.apache.hadoop.hbase.filter.PrefixFilter;
80  import org.apache.hadoop.hbase.filter.QualifierFilter;
81  import org.apache.hadoop.hbase.filter.RegexStringComparator;
82  import org.apache.hadoop.hbase.filter.RowFilter;
83  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
84  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
85  import org.apache.hadoop.hbase.io.hfile.BlockCache;
86  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
87  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
88  import org.apache.hadoop.hbase.master.HMaster;
89  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
90  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
91  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
92  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
93  import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
94  import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
95  import org.apache.hadoop.hbase.regionserver.HRegion;
96  import org.apache.hadoop.hbase.regionserver.HRegionServer;
97  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
98  import org.apache.hadoop.hbase.regionserver.Region;
99  import org.apache.hadoop.hbase.regionserver.Store;
100 import org.apache.hadoop.hbase.testclassification.LargeTests;
101 import org.apache.hadoop.hbase.util.Bytes;
102 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
103 import org.apache.hadoop.hbase.util.Pair;
104 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
105 import org.apache.log4j.AppenderSkeleton;
106 import org.apache.log4j.Logger;
107 import org.apache.log4j.spi.LoggingEvent;
108 import org.junit.After;
109 import org.junit.AfterClass;
110 import org.junit.Before;
111 import org.junit.BeforeClass;
112 import org.junit.Ignore;
113 import org.junit.Test;
114 import org.junit.experimental.categories.Category;
115 /**
116  * Run tests that use the HBase clients; {@link HTable}.
117  * Sets up the HBase mini cluster once at start and runs through all client tests.
118  * Each creates a table named for the method and does its stuff against that.
119  */
120 @Category(LargeTests.class)
121 @SuppressWarnings ("deprecation")
122 public class TestFromClientSide {
123   // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide.
124   final Log LOG = LogFactory.getLog(getClass());
125   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
126   private static byte [] ROW = Bytes.toBytes("testRow");
127   private static byte [] FAMILY = Bytes.toBytes("testFamily");
128   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
129   private static byte [] VALUE = Bytes.toBytes("testValue");
130   protected static int SLAVES = 3;
131 
132   /**
133    * @throws java.lang.Exception
134    */
135   @BeforeClass
136   public static void setUpBeforeClass() throws Exception {
137     // Uncomment the following lines if more verbosity is needed for
138     // debugging (see HBASE-12285 for details).
139     //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
140     //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
141     //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
142     Configuration conf = TEST_UTIL.getConfiguration();
143     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
144         MultiRowMutationEndpoint.class.getName());
145     conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
146     // We need more than one region server in this test
147     TEST_UTIL.startMiniCluster(SLAVES);
148   }
149 
150   /**
151    * @throws java.lang.Exception
152    */
153   @AfterClass
154   public static void tearDownAfterClass() throws Exception {
155     TEST_UTIL.shutdownMiniCluster();
156   }
157 
158   /**
159    * @throws java.lang.Exception
160    */
161   @Before
162   public void setUp() throws Exception {
163     // Nothing to do.
164   }
165 
166   /**
167    * @throws java.lang.Exception
168    */
169   @After
170   public void tearDown() throws Exception {
171     // Nothing to do.
172   }
173 
174   /**
175    * Basic client side validation of HBASE-4536
176    */
177    @Test
178    public void testKeepDeletedCells() throws Exception {
179      final TableName TABLENAME = TableName.valueOf("testKeepDeletesCells");
180      final byte[] FAMILY = Bytes.toBytes("family");
181      final byte[] C0 = Bytes.toBytes("c0");
182 
183      final byte[] T1 = Bytes.toBytes("T1");
184      final byte[] T2 = Bytes.toBytes("T2");
185      final byte[] T3 = Bytes.toBytes("T3");
186      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
187          .setKeepDeletedCells(true).setMaxVersions(3);
188 
189      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
190      desc.addFamily(hcd);
191      TEST_UTIL.getHBaseAdmin().createTable(desc);
192      Configuration c = TEST_UTIL.getConfiguration();
193      Table h = new HTable(c, TABLENAME);
194 
195      long ts = System.currentTimeMillis();
196      Put p = new Put(T1, ts);
197      p.add(FAMILY, C0, T1);
198      h.put(p);
199      p = new Put(T1, ts+2);
200      p.add(FAMILY, C0, T2);
201      h.put(p);
202      p = new Put(T1, ts+4);
203      p.add(FAMILY, C0, T3);
204      h.put(p);
205 
206      Delete d = new Delete(T1, ts+3);
207      h.delete(d);
208 
209      d = new Delete(T1, ts+3);
210      d.deleteColumns(FAMILY, C0, ts+3);
211      h.delete(d);
212 
213      Get g = new Get(T1);
214      // does *not* include the delete
215      g.setTimeRange(0, ts+3);
216      Result r = h.get(g);
217      assertArrayEquals(T2, r.getValue(FAMILY, C0));
218 
219      Scan s = new Scan(T1);
220      s.setTimeRange(0, ts+3);
221      s.setMaxVersions();
222      ResultScanner scanner = h.getScanner(s);
223      Cell[] kvs = scanner.next().rawCells();
224      assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
225      assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
226      scanner.close();
227 
228      s = new Scan(T1);
229      s.setRaw(true);
230      s.setMaxVersions();
231      scanner = h.getScanner(s);
232      kvs = scanner.next().rawCells();
233      assertTrue(CellUtil.isDeleteFamily(kvs[0]));
234      assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
235      assertTrue(CellUtil.isDelete(kvs[2]));
236      assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
237      assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
238      scanner.close();
239      h.close();
240    }
241 
242     /**
243     * Basic client side validation of HBASE-10118
244     */
245    @Test
246    public void testPurgeFutureDeletes() throws Exception {
247      final TableName TABLENAME = TableName.valueOf("testPurgeFutureDeletes");
248      final byte[] ROW = Bytes.toBytes("row");
249      final byte[] FAMILY = Bytes.toBytes("family");
250      final byte[] COLUMN = Bytes.toBytes("column");
251      final byte[] VALUE = Bytes.toBytes("value");
252 
253      Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
254 
255      // future timestamp
256      long ts = System.currentTimeMillis() * 2;
257      Put put = new Put(ROW, ts);
258      put.add(FAMILY, COLUMN, VALUE);
259      table.put(put);
260 
261      Get get = new Get(ROW);
262      Result result = table.get(get);
263      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
264 
265      Delete del = new Delete(ROW);
266      del.deleteColumn(FAMILY, COLUMN, ts);
267      table.delete(del);
268 
269      get = new Get(ROW);
270      result = table.get(get);
271      assertNull(result.getValue(FAMILY, COLUMN));
272 
273      // major compaction, purged future deletes
274      TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
275      TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME);
276 
277      // waiting for the major compaction to complete
278      TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
279        @Override
280        public boolean evaluate() throws IOException {
281          return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) ==
282              AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
283        }
284      });
285 
286      put = new Put(ROW, ts);
287      put.add(FAMILY, COLUMN, VALUE);
288      table.put(put);
289 
290      get = new Get(ROW);
291      result = table.get(get);
292      assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
293 
294      table.close();
295    }
296 
297    /**
298     * @deprecated Tests deprecated functionality. Remove when we are past 1.0.
299     * @throws Exception
300     */
301    @Deprecated
302    @Test
303    public void testSharedZooKeeper() throws Exception {
304      Configuration newConfig = new Configuration(TEST_UTIL.getConfiguration());
305      newConfig.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "12345");
306 
307      // First with a simple ZKW
308      ZooKeeperWatcher z0 = new ZooKeeperWatcher(
309        newConfig, "hconnection", new Abortable() {
310        @Override public void abort(String why, Throwable e) {}
311        @Override public boolean isAborted() {return false;}
312      });
313      z0.getRecoverableZooKeeper().getZooKeeper().exists("/oldZooKeeperWatcher", false);
314      z0.close();
315 
316      // Then a ZooKeeperKeepAliveConnection
317      ConnectionManager.HConnectionImplementation connection1 =
318        (ConnectionManager.HConnectionImplementation)
319          HConnectionManager.getConnection(newConfig);
320 
321      ZooKeeperKeepAliveConnection z1 = connection1.getKeepAliveZooKeeperWatcher();
322      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1", false);
323 
324      z1.close();
325 
326      // will still work, because the real connection is not closed yet
327      // Not do be done in real code
328      z1.getRecoverableZooKeeper().getZooKeeper().exists("/z1afterclose", false);
329 
330 
331      ZooKeeperKeepAliveConnection z2 = connection1.getKeepAliveZooKeeperWatcher();
332      assertTrue(
333        "ZooKeeperKeepAliveConnection equals on same connection", z1 == z2);
334 
335 
336 
337      Configuration newConfig2 = new Configuration(TEST_UTIL.getConfiguration());
338      newConfig2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "6789");
339      ConnectionManager.HConnectionImplementation connection2 =
340        (ConnectionManager.HConnectionImplementation)
341          HConnectionManager.getConnection(newConfig2);
342 
343      assertTrue("connections should be different ", connection1 != connection2);
344 
345      ZooKeeperKeepAliveConnection z3 = connection2.getKeepAliveZooKeeperWatcher();
346      assertTrue(
347        "ZooKeeperKeepAliveConnection should be different" +
348          " on different connections", z1 != z3);
349 
350      // Bypass the private access
351      Method m = ConnectionManager.HConnectionImplementation.class.
352        getDeclaredMethod("closeZooKeeperWatcher");
353      m.setAccessible(true);
354      m.invoke(connection2);
355 
356      ZooKeeperKeepAliveConnection z4 = connection2.getKeepAliveZooKeeperWatcher();
357      assertTrue(
358        "ZooKeeperKeepAliveConnection should be recreated" +
359          " when previous connections was closed"
360        , z3 != z4);
361 
362 
363      z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
364      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
365 
366 
367      HConnectionManager.deleteConnection(newConfig);
368      try {
369        z2.getRecoverableZooKeeper().getZooKeeper().exists("/z2", false);
370        assertTrue("We should not have a valid connection for z2", false);
371      } catch (Exception e){
372      }
373 
374      z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
375      // We expect success here.
376 
377 
378      HConnectionManager.deleteConnection(newConfig2);
379      try {
380        z4.getRecoverableZooKeeper().getZooKeeper().exists("/z4", false);
381        assertTrue("We should not have a valid connection for z4", false);
382      } catch (Exception e){
383      }
384    }
385 
386 
387   /**
388    * Verifies that getConfiguration returns the same Configuration object used
389    * to create the HTable instance.
390    */
391   @Test
392   public void testGetConfiguration() throws Exception {
393     TableName TABLE = TableName.valueOf("testGetConfiguration");
394     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
395     Configuration conf = TEST_UTIL.getConfiguration();
396     Table table = TEST_UTIL.createTable(TABLE, FAMILIES, conf);
397     assertSame(conf, table.getConfiguration());
398   }
399 
400   /**
401    * Test from client side of an involved filter against a multi family that
402    * involves deletes.
403    *
404    * @throws Exception
405    */
406   @Test
407   public void testWeirdCacheBehaviour() throws Exception {
408     TableName TABLE = TableName.valueOf("testWeirdCacheBehaviour");
409     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
410         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
411         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
412     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES);
413     String value = "this is the value";
414     String value2 = "this is some other value";
415     String keyPrefix1 = UUID.randomUUID().toString();
416     String keyPrefix2 = UUID.randomUUID().toString();
417     String keyPrefix3 = UUID.randomUUID().toString();
418     putRows(ht, 3, value, keyPrefix1);
419     putRows(ht, 3, value, keyPrefix2);
420     putRows(ht, 3, value, keyPrefix3);
421     ht.flushCommits();
422     putRows(ht, 3, value2, keyPrefix1);
423     putRows(ht, 3, value2, keyPrefix2);
424     putRows(ht, 3, value2, keyPrefix3);
425     Table table = new HTable(TEST_UTIL.getConfiguration(), TABLE);
426     System.out.println("Checking values for key: " + keyPrefix1);
427     assertEquals("Got back incorrect number of rows from scan", 3,
428         getNumberOfRows(keyPrefix1, value2, table));
429     System.out.println("Checking values for key: " + keyPrefix2);
430     assertEquals("Got back incorrect number of rows from scan", 3,
431         getNumberOfRows(keyPrefix2, value2, table));
432     System.out.println("Checking values for key: " + keyPrefix3);
433     assertEquals("Got back incorrect number of rows from scan", 3,
434         getNumberOfRows(keyPrefix3, value2, table));
435     deleteColumns(ht, value2, keyPrefix1);
436     deleteColumns(ht, value2, keyPrefix2);
437     deleteColumns(ht, value2, keyPrefix3);
438     System.out.println("Starting important checks.....");
439     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
440       0, getNumberOfRows(keyPrefix1, value2, table));
441     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
442       0, getNumberOfRows(keyPrefix2, value2, table));
443     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
444       0, getNumberOfRows(keyPrefix3, value2, table));
445     ht.setScannerCaching(0);
446     assertEquals("Got back incorrect number of rows from scan", 0,
447       getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
448     assertEquals("Got back incorrect number of rows from scan", 0,
449       getNumberOfRows(keyPrefix2, value2, table));
450   }
451 
452   private void deleteColumns(Table ht, String value, String keyPrefix)
453   throws IOException {
454     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
455     Iterator<Result> it = scanner.iterator();
456     int count = 0;
457     while (it.hasNext()) {
458       Result result = it.next();
459       Delete delete = new Delete(result.getRow());
460       delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
461       ht.delete(delete);
462       count++;
463     }
464     assertEquals("Did not perform correct number of deletes", 3, count);
465   }
466 
467   private int getNumberOfRows(String keyPrefix, String value, Table ht)
468       throws Exception {
469     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
470     Iterator<Result> scanner = resultScanner.iterator();
471     int numberOfResults = 0;
472     while (scanner.hasNext()) {
473       Result result = scanner.next();
474       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
475       for (Cell kv : result.rawCells()) {
476         System.out.println("kv=" + kv.toString() + ", "
477             + Bytes.toString(CellUtil.cloneValue(kv)));
478       }
479       numberOfResults++;
480     }
481     return numberOfResults;
482   }
483 
484   private ResultScanner buildScanner(String keyPrefix, String value, Table ht)
485       throws IOException {
486     // OurFilterList allFilters = new OurFilterList();
487     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
488     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
489     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
490         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
491         .toBytes(value));
492     filter.setFilterIfMissing(true);
493     allFilters.addFilter(filter);
494 
495     // allFilters.addFilter(new
496     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
497     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
498 
499     Scan scan = new Scan();
500     scan.addFamily(Bytes.toBytes("trans-blob"));
501     scan.addFamily(Bytes.toBytes("trans-type"));
502     scan.addFamily(Bytes.toBytes("trans-date"));
503     scan.addFamily(Bytes.toBytes("trans-tags"));
504     scan.addFamily(Bytes.toBytes("trans-group"));
505     scan.setFilter(allFilters);
506 
507     return ht.getScanner(scan);
508   }
509 
510   private void putRows(Table ht, int numRows, String value, String key)
511       throws IOException {
512     for (int i = 0; i < numRows; i++) {
513       String row = key + "_" + UUID.randomUUID().toString();
514       System.out.println(String.format("Saving row: %s, with value %s", row,
515           value));
516       Put put = new Put(Bytes.toBytes(row));
517       put.setDurability(Durability.SKIP_WAL);
518       put.add(Bytes.toBytes("trans-blob"), null, Bytes
519           .toBytes("value for blob"));
520       put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
521       put.add(Bytes.toBytes("trans-date"), null, Bytes
522           .toBytes("20090921010101999"));
523       put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
524           .toBytes(value));
525       put.add(Bytes.toBytes("trans-group"), null, Bytes
526           .toBytes("adhocTransactionGroupId"));
527       ht.put(put);
528     }
529   }
530 
531   /**
532    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
533    * logs to ensure that we're not scanning more regions that we're supposed to.
534    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
535    * @throws IOException
536    * @throws InterruptedException
537    */
538   @Test
539   public void testFilterAcrossMultipleRegions()
540   throws IOException, InterruptedException {
541     TableName name = TableName.valueOf("testFilterAcrossMutlipleRegions");
542     HTable t = TEST_UTIL.createTable(name, FAMILY);
543     int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
544     assertRowCount(t, rowCount);
545     // Split the table.  Should split on a reasonable key; 'lqj'
546     Map<HRegionInfo, ServerName> regions  = splitTable(t);
547     assertRowCount(t, rowCount);
548     // Get end key of first region.
549     byte [] endKey = regions.keySet().iterator().next().getEndKey();
550     // Count rows with a filter that stops us before passed 'endKey'.
551     // Should be count of rows in first region.
552     int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
553     assertTrue(endKeyCount < rowCount);
554 
555     // How do I know I did not got to second region?  Thats tough.  Can't really
556     // do that in client-side region test.  I verified by tracing in debugger.
557     // I changed the messages that come out when set to DEBUG so should see
558     // when scanner is done. Says "Finished with scanning..." with region name.
559     // Check that its finished in right region.
560 
561     // New test.  Make it so scan goes into next region by one and then two.
562     // Make sure count comes out right.
563     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
564     int plusOneCount = countRows(t, createScanWithRowFilter(key));
565     assertEquals(endKeyCount + 1, plusOneCount);
566     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
567     int plusTwoCount = countRows(t, createScanWithRowFilter(key));
568     assertEquals(endKeyCount + 2, plusTwoCount);
569 
570     // New test.  Make it so I scan one less than endkey.
571     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
572     int minusOneCount = countRows(t, createScanWithRowFilter(key));
573     assertEquals(endKeyCount - 1, minusOneCount);
574     // For above test... study logs.  Make sure we do "Finished with scanning.."
575     // in first region and that we do not fall into the next region.
576 
577     key = new byte [] {'a', 'a', 'a'};
578     int countBBB = countRows(t,
579       createScanWithRowFilter(key, null, CompareFilter.CompareOp.EQUAL));
580     assertEquals(1, countBBB);
581 
582     int countGreater = countRows(t, createScanWithRowFilter(endKey, null,
583       CompareFilter.CompareOp.GREATER_OR_EQUAL));
584     // Because started at start of table.
585     assertEquals(0, countGreater);
586     countGreater = countRows(t, createScanWithRowFilter(endKey, endKey,
587       CompareFilter.CompareOp.GREATER_OR_EQUAL));
588     assertEquals(rowCount - endKeyCount, countGreater);
589   }
590 
591   /*
592    * @param key
593    * @return Scan with RowFilter that does LESS than passed key.
594    */
595   private Scan createScanWithRowFilter(final byte [] key) {
596     return createScanWithRowFilter(key, null, CompareFilter.CompareOp.LESS);
597   }
598 
599   /*
600    * @param key
601    * @param op
602    * @param startRow
603    * @return Scan with RowFilter that does CompareOp op on passed key.
604    */
605   private Scan createScanWithRowFilter(final byte [] key,
606       final byte [] startRow, CompareFilter.CompareOp op) {
607     // Make sure key is of some substance... non-null and > than first key.
608     assertTrue(key != null && key.length > 0 &&
609       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
610     LOG.info("Key=" + Bytes.toString(key));
611     Scan s = startRow == null? new Scan(): new Scan(startRow);
612     Filter f = new RowFilter(op, new BinaryComparator(key));
613     f = new WhileMatchFilter(f);
614     s.setFilter(f);
615     return s;
616   }
617 
618   /*
619    * @param t
620    * @param s
621    * @return Count of rows in table.
622    * @throws IOException
623    */
624   private int countRows(final Table t, final Scan s)
625   throws IOException {
626     // Assert all rows in table.
627     ResultScanner scanner = t.getScanner(s);
628     int count = 0;
629     for (Result result: scanner) {
630       count++;
631       assertTrue(result.size() > 0);
632       // LOG.info("Count=" + count + ", row=" + Bytes.toString(result.getRow()));
633     }
634     return count;
635   }
636 
637   private void assertRowCount(final Table t, final int expected)
638   throws IOException {
639     assertEquals(expected, countRows(t, new Scan()));
640   }
641 
642   /*
643    * Split table into multiple regions.
644    * @param t Table to split.
645    * @return Map of regions to servers.
646    * @throws IOException
647    */
648   private Map<HRegionInfo, ServerName> splitTable(final HTable t)
649   throws IOException, InterruptedException {
650     // Split this table in two.
651     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
652     admin.split(t.getTableName());
653     admin.close();
654     Map<HRegionInfo, ServerName> regions = waitOnSplit(t);
655     assertTrue(regions.size() > 1);
656     return regions;
657   }
658 
659   /*
660    * Wait on table split.  May return because we waited long enough on the split
661    * and it didn't happen.  Caller should check.
662    * @param t
663    * @return Map of table regions; caller needs to check table actually split.
664    */
665   private Map<HRegionInfo, ServerName> waitOnSplit(final HTable t)
666   throws IOException {
667     Map<HRegionInfo, ServerName> regions = t.getRegionLocations();
668     int originalCount = regions.size();
669     for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
670       Thread.currentThread();
671       try {
672         Thread.sleep(1000);
673       } catch (InterruptedException e) {
674         e.printStackTrace();
675       }
676       regions = t.getRegionLocations();
677       if (regions.size() > originalCount) break;
678     }
679     return regions;
680   }
681 
682   @Test
683   public void testSuperSimple() throws Exception {
684     byte [] TABLE = Bytes.toBytes("testSuperSimple");
685     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
686     Put put = new Put(ROW);
687     put.add(FAMILY, QUALIFIER, VALUE);
688     ht.put(put);
689     Scan scan = new Scan();
690     scan.addColumn(FAMILY, TABLE);
691     ResultScanner scanner = ht.getScanner(scan);
692     Result result = scanner.next();
693     assertTrue("Expected null result", result == null);
694     scanner.close();
695   }
696 
697   @Test
698   public void testMaxKeyValueSize() throws Exception {
699     byte [] TABLE = Bytes.toBytes("testMaxKeyValueSize");
700     Configuration conf = TEST_UTIL.getConfiguration();
701     String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY);
702     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
703     byte[] value = new byte[4 * 1024 * 1024];
704     Put put = new Put(ROW);
705     put.add(FAMILY, QUALIFIER, value);
706     ht.put(put);
707     try {
708       TEST_UTIL.getConfiguration().setInt(
709           ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024);
710       // Create new table so we pick up the change in Configuration.
711       try (Connection connection =
712           ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
713         try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
714           put = new Put(ROW);
715           put.add(FAMILY, QUALIFIER, value);
716           t.put(put);
717         }
718       }
719       fail("Inserting a too large KeyValue worked, should throw exception");
720     } catch(Exception e) {}
721     conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize);
722   }
723 
724   @Test
725   public void testFilters() throws Exception {
726     byte [] TABLE = Bytes.toBytes("testFilters");
727     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
728     byte [][] ROWS = makeN(ROW, 10);
729     byte [][] QUALIFIERS = {
730         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
731         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
732         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
733         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
734         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
735     };
736     for(int i=0;i<10;i++) {
737       Put put = new Put(ROWS[i]);
738       put.setDurability(Durability.SKIP_WAL);
739       put.add(FAMILY, QUALIFIERS[i], VALUE);
740       ht.put(put);
741     }
742     Scan scan = new Scan();
743     scan.addFamily(FAMILY);
744     Filter filter = new QualifierFilter(CompareOp.EQUAL,
745       new RegexStringComparator("col[1-5]"));
746     scan.setFilter(filter);
747     ResultScanner scanner = ht.getScanner(scan);
748     int expectedIndex = 1;
749     for(Result result : ht.getScanner(scan)) {
750       assertEquals(result.size(), 1);
751       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
752       assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
753           QUALIFIERS[expectedIndex]));
754       expectedIndex++;
755     }
756     assertEquals(expectedIndex, 6);
757     scanner.close();
758   }
759 
760   @Test
761   public void testFilterWithLongCompartor() throws Exception {
762     byte [] TABLE = Bytes.toBytes("testFilterWithLongCompartor");
763     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
764     byte [][] ROWS = makeN(ROW, 10);
765     byte [][] values = new byte[10][];
766     for (int i = 0; i < 10; i ++) {
767         values[i] = Bytes.toBytes(100L * i);
768     }
769     for(int i = 0; i < 10; i ++) {
770       Put put = new Put(ROWS[i]);
771       put.setDurability(Durability.SKIP_WAL);
772       put.add(FAMILY, QUALIFIER, values[i]);
773       ht.put(put);
774     }
775     Scan scan = new Scan();
776     scan.addFamily(FAMILY);
777     Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER,
778       new LongComparator(500));
779     scan.setFilter(filter);
780     ResultScanner scanner = ht.getScanner(scan);
781     int expectedIndex = 0;
782     for(Result result : ht.getScanner(scan)) {
783       assertEquals(result.size(), 1);
784       assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
785       expectedIndex++;
786     }
787     assertEquals(expectedIndex, 4);
788     scanner.close();
789 }
790 
791   @Test
792   public void testKeyOnlyFilter() throws Exception {
793     byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
794     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
795     byte [][] ROWS = makeN(ROW, 10);
796     byte [][] QUALIFIERS = {
797         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
798         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
799         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
800         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
801         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
802     };
803     for(int i=0;i<10;i++) {
804       Put put = new Put(ROWS[i]);
805       put.setDurability(Durability.SKIP_WAL);
806       put.add(FAMILY, QUALIFIERS[i], VALUE);
807       ht.put(put);
808     }
809     Scan scan = new Scan();
810     scan.addFamily(FAMILY);
811     Filter filter = new KeyOnlyFilter(true);
812     scan.setFilter(filter);
813     ResultScanner scanner = ht.getScanner(scan);
814     int count = 0;
815     for(Result result : ht.getScanner(scan)) {
816       assertEquals(result.size(), 1);
817       assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
818       assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
819       count++;
820     }
821     assertEquals(count, 10);
822     scanner.close();
823   }
824 
825   /**
826    * Test simple table and non-existent row cases.
827    */
828   @Test
829   public void testSimpleMissing() throws Exception {
830     byte [] TABLE = Bytes.toBytes("testSimpleMissing");
831     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
832     byte [][] ROWS = makeN(ROW, 4);
833 
834     // Try to get a row on an empty table
835     Get get = new Get(ROWS[0]);
836     Result result = ht.get(get);
837     assertEmptyResult(result);
838 
839     get = new Get(ROWS[0]);
840     get.addFamily(FAMILY);
841     result = ht.get(get);
842     assertEmptyResult(result);
843 
844     get = new Get(ROWS[0]);
845     get.addColumn(FAMILY, QUALIFIER);
846     result = ht.get(get);
847     assertEmptyResult(result);
848 
849     Scan scan = new Scan();
850     result = getSingleScanResult(ht, scan);
851     assertNullResult(result);
852 
853 
854     scan = new Scan(ROWS[0]);
855     result = getSingleScanResult(ht, scan);
856     assertNullResult(result);
857 
858     scan = new Scan(ROWS[0],ROWS[1]);
859     result = getSingleScanResult(ht, scan);
860     assertNullResult(result);
861 
862     scan = new Scan();
863     scan.addFamily(FAMILY);
864     result = getSingleScanResult(ht, scan);
865     assertNullResult(result);
866 
867     scan = new Scan();
868     scan.addColumn(FAMILY, QUALIFIER);
869     result = getSingleScanResult(ht, scan);
870     assertNullResult(result);
871 
872     // Insert a row
873 
874     Put put = new Put(ROWS[2]);
875     put.add(FAMILY, QUALIFIER, VALUE);
876     ht.put(put);
877 
878     // Try to get empty rows around it
879 
880     get = new Get(ROWS[1]);
881     result = ht.get(get);
882     assertEmptyResult(result);
883 
884     get = new Get(ROWS[0]);
885     get.addFamily(FAMILY);
886     result = ht.get(get);
887     assertEmptyResult(result);
888 
889     get = new Get(ROWS[3]);
890     get.addColumn(FAMILY, QUALIFIER);
891     result = ht.get(get);
892     assertEmptyResult(result);
893 
894     // Try to scan empty rows around it
895 
896     scan = new Scan(ROWS[3]);
897     result = getSingleScanResult(ht, scan);
898     assertNullResult(result);
899 
900     scan = new Scan(ROWS[0],ROWS[2]);
901     result = getSingleScanResult(ht, scan);
902     assertNullResult(result);
903 
904     // Make sure we can actually get the row
905 
906     get = new Get(ROWS[2]);
907     result = ht.get(get);
908     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
909 
910     get = new Get(ROWS[2]);
911     get.addFamily(FAMILY);
912     result = ht.get(get);
913     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
914 
915     get = new Get(ROWS[2]);
916     get.addColumn(FAMILY, QUALIFIER);
917     result = ht.get(get);
918     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
919 
920     // Make sure we can scan the row
921 
922     scan = new Scan();
923     result = getSingleScanResult(ht, scan);
924     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
925 
926     scan = new Scan(ROWS[0],ROWS[3]);
927     result = getSingleScanResult(ht, scan);
928     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
929 
930     scan = new Scan(ROWS[2],ROWS[3]);
931     result = getSingleScanResult(ht, scan);
932     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
933   }
934 
935   /**
936    * Test basic puts, gets, scans, and deletes for a single row
937    * in a multiple family table.
938    */
939   @Test
940   public void testSingleRowMultipleFamily() throws Exception {
941     byte [] TABLE = Bytes.toBytes("testSingleRowMultipleFamily");
942     byte [][] ROWS = makeN(ROW, 3);
943     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
944     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
945     byte [][] VALUES = makeN(VALUE, 10);
946 
947     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES);
948 
949     Get get;
950     Scan scan;
951     Delete delete;
952     Put put;
953     Result result;
954 
955     ////////////////////////////////////////////////////////////////////////////
956     // Insert one column to one family
957     ////////////////////////////////////////////////////////////////////////////
958 
959     put = new Put(ROWS[0]);
960     put.add(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
961     ht.put(put);
962 
963     // Get the single column
964     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
965 
966     // Scan the single column
967     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
968 
969     // Get empty results around inserted column
970     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
971 
972     // Scan empty results around inserted column
973     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
974 
975     ////////////////////////////////////////////////////////////////////////////
976     // Flush memstore and run same tests from storefiles
977     ////////////////////////////////////////////////////////////////////////////
978 
979     TEST_UTIL.flush();
980 
981     // Redo get and scan tests from storefile
982     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
983     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
984     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
985     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
986 
987     ////////////////////////////////////////////////////////////////////////////
988     // Now, Test reading from memstore and storefiles at once
989     ////////////////////////////////////////////////////////////////////////////
990 
991     // Insert multiple columns to two other families
992     put = new Put(ROWS[0]);
993     put.add(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
994     put.add(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
995     put.add(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
996     put.add(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
997     put.add(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
998     put.add(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
999     put.add(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
1000     ht.put(put);
1001 
1002     // Get multiple columns across multiple families and get empties around it
1003     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1004 
1005     // Scan multiple columns across multiple families and scan empties around it
1006     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1007 
1008     ////////////////////////////////////////////////////////////////////////////
1009     // Flush the table again
1010     ////////////////////////////////////////////////////////////////////////////
1011 
1012     TEST_UTIL.flush();
1013 
1014     // Redo tests again
1015     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1016     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1017 
1018     // Insert more data to memstore
1019     put = new Put(ROWS[0]);
1020     put.add(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
1021     put.add(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1022     put.add(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1023     put.add(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
1024     ht.put(put);
1025 
1026     ////////////////////////////////////////////////////////////////////////////
1027     // Delete a storefile column
1028     ////////////////////////////////////////////////////////////////////////////
1029     delete = new Delete(ROWS[0]);
1030     delete.deleteColumns(FAMILIES[6], QUALIFIERS[7]);
1031     ht.delete(delete);
1032 
1033     // Try to get deleted column
1034     get = new Get(ROWS[0]);
1035     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
1036     result = ht.get(get);
1037     assertEmptyResult(result);
1038 
1039     // Try to scan deleted column
1040     scan = new Scan();
1041     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
1042     result = getSingleScanResult(ht, scan);
1043     assertNullResult(result);
1044 
1045     // Make sure we can still get a column before it and after it
1046     get = new Get(ROWS[0]);
1047     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1048     result = ht.get(get);
1049     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1050 
1051     get = new Get(ROWS[0]);
1052     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1053     result = ht.get(get);
1054     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1055 
1056     // Make sure we can still scan a column before it and after it
1057     scan = new Scan();
1058     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1059     result = getSingleScanResult(ht, scan);
1060     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1061 
1062     scan = new Scan();
1063     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1064     result = getSingleScanResult(ht, scan);
1065     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1066 
1067     ////////////////////////////////////////////////////////////////////////////
1068     // Delete a memstore column
1069     ////////////////////////////////////////////////////////////////////////////
1070     delete = new Delete(ROWS[0]);
1071     delete.deleteColumns(FAMILIES[6], QUALIFIERS[8]);
1072     ht.delete(delete);
1073 
1074     // Try to get deleted column
1075     get = new Get(ROWS[0]);
1076     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1077     result = ht.get(get);
1078     assertEmptyResult(result);
1079 
1080     // Try to scan deleted column
1081     scan = new Scan();
1082     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1083     result = getSingleScanResult(ht, scan);
1084     assertNullResult(result);
1085 
1086     // Make sure we can still get a column before it and after it
1087     get = new Get(ROWS[0]);
1088     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1089     result = ht.get(get);
1090     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1091 
1092     get = new Get(ROWS[0]);
1093     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1094     result = ht.get(get);
1095     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1096 
1097     // Make sure we can still scan a column before it and after it
1098     scan = new Scan();
1099     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1100     result = getSingleScanResult(ht, scan);
1101     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1102 
1103     scan = new Scan();
1104     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1105     result = getSingleScanResult(ht, scan);
1106     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1107 
1108     ////////////////////////////////////////////////////////////////////////////
1109     // Delete joint storefile/memstore family
1110     ////////////////////////////////////////////////////////////////////////////
1111 
1112     delete = new Delete(ROWS[0]);
1113     delete.deleteFamily(FAMILIES[4]);
1114     ht.delete(delete);
1115 
1116     // Try to get storefile column in deleted family
1117     get = new Get(ROWS[0]);
1118     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1119     result = ht.get(get);
1120     assertEmptyResult(result);
1121 
1122     // Try to get memstore column in deleted family
1123     get = new Get(ROWS[0]);
1124     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1125     result = ht.get(get);
1126     assertEmptyResult(result);
1127 
1128     // Try to get deleted family
1129     get = new Get(ROWS[0]);
1130     get.addFamily(FAMILIES[4]);
1131     result = ht.get(get);
1132     assertEmptyResult(result);
1133 
1134     // Try to scan storefile column in deleted family
1135     scan = new Scan();
1136     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1137     result = getSingleScanResult(ht, scan);
1138     assertNullResult(result);
1139 
1140     // Try to scan memstore column in deleted family
1141     scan = new Scan();
1142     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1143     result = getSingleScanResult(ht, scan);
1144     assertNullResult(result);
1145 
1146     // Try to scan deleted family
1147     scan = new Scan();
1148     scan.addFamily(FAMILIES[4]);
1149     result = getSingleScanResult(ht, scan);
1150     assertNullResult(result);
1151 
1152     // Make sure we can still get another family
1153     get = new Get(ROWS[0]);
1154     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1155     result = ht.get(get);
1156     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1157 
1158     get = new Get(ROWS[0]);
1159     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1160     result = ht.get(get);
1161     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1162 
1163     // Make sure we can still scan another family
1164     scan = new Scan();
1165     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1166     result = getSingleScanResult(ht, scan);
1167     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1168 
1169     scan = new Scan();
1170     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1171     result = getSingleScanResult(ht, scan);
1172     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1173 
1174     ////////////////////////////////////////////////////////////////////////////
1175     // Flush everything and rerun delete tests
1176     ////////////////////////////////////////////////////////////////////////////
1177 
1178     TEST_UTIL.flush();
1179 
1180     // Try to get storefile column in deleted family
1181     get = new Get(ROWS[0]);
1182     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1183     result = ht.get(get);
1184     assertEmptyResult(result);
1185 
1186     // Try to get memstore column in deleted family
1187     get = new Get(ROWS[0]);
1188     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1189     result = ht.get(get);
1190     assertEmptyResult(result);
1191 
1192     // Try to get deleted family
1193     get = new Get(ROWS[0]);
1194     get.addFamily(FAMILIES[4]);
1195     result = ht.get(get);
1196     assertEmptyResult(result);
1197 
1198     // Try to scan storefile column in deleted family
1199     scan = new Scan();
1200     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1201     result = getSingleScanResult(ht, scan);
1202     assertNullResult(result);
1203 
1204     // Try to scan memstore column in deleted family
1205     scan = new Scan();
1206     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1207     result = getSingleScanResult(ht, scan);
1208     assertNullResult(result);
1209 
1210     // Try to scan deleted family
1211     scan = new Scan();
1212     scan.addFamily(FAMILIES[4]);
1213     result = getSingleScanResult(ht, scan);
1214     assertNullResult(result);
1215 
1216     // Make sure we can still get another family
1217     get = new Get(ROWS[0]);
1218     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1219     result = ht.get(get);
1220     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1221 
1222     get = new Get(ROWS[0]);
1223     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1224     result = ht.get(get);
1225     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1226 
1227     // Make sure we can still scan another family
1228     scan = new Scan();
1229     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1230     result = getSingleScanResult(ht, scan);
1231     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1232 
1233     scan = new Scan();
1234     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1235     result = getSingleScanResult(ht, scan);
1236     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1237 
1238   }
1239 
1240   @Test
1241   public void testNull() throws Exception {
1242     byte [] TABLE = Bytes.toBytes("testNull");
1243 
1244     // Null table name (should NOT work)
1245     try {
1246       TEST_UTIL.createTable((TableName)null, FAMILY);
1247       fail("Creating a table with null name passed, should have failed");
1248     } catch(Exception e) {}
1249 
1250     // Null family (should NOT work)
1251     try {
1252       TEST_UTIL.createTable(TABLE, new byte[][]{(byte[])null});
1253       fail("Creating a table with a null family passed, should fail");
1254     } catch(Exception e) {}
1255 
1256     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
1257 
1258     // Null row (should NOT work)
1259     try {
1260       Put put = new Put((byte[])null);
1261       put.add(FAMILY, QUALIFIER, VALUE);
1262       ht.put(put);
1263       fail("Inserting a null row worked, should throw exception");
1264     } catch(Exception e) {}
1265 
1266     // Null qualifier (should work)
1267     {
1268       Put put = new Put(ROW);
1269       put.add(FAMILY, null, VALUE);
1270       ht.put(put);
1271 
1272       getTestNull(ht, ROW, FAMILY, VALUE);
1273 
1274       scanTestNull(ht, ROW, FAMILY, VALUE);
1275 
1276       Delete delete = new Delete(ROW);
1277       delete.deleteColumns(FAMILY, null);
1278       ht.delete(delete);
1279 
1280       Get get = new Get(ROW);
1281       Result result = ht.get(get);
1282       assertEmptyResult(result);
1283     }
1284 
1285     // Use a new table
1286     byte [] TABLE2 = Bytes.toBytes("testNull2");
1287     ht = TEST_UTIL.createTable(TableName.valueOf(TABLE2), FAMILY);
1288 
1289     // Empty qualifier, byte[0] instead of null (should work)
1290     try {
1291       Put put = new Put(ROW);
1292       put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1293       ht.put(put);
1294 
1295       getTestNull(ht, ROW, FAMILY, VALUE);
1296 
1297       scanTestNull(ht, ROW, FAMILY, VALUE);
1298 
1299       // Flush and try again
1300 
1301       TEST_UTIL.flush();
1302 
1303       getTestNull(ht, ROW, FAMILY, VALUE);
1304 
1305       scanTestNull(ht, ROW, FAMILY, VALUE);
1306 
1307       Delete delete = new Delete(ROW);
1308       delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1309       ht.delete(delete);
1310 
1311       Get get = new Get(ROW);
1312       Result result = ht.get(get);
1313       assertEmptyResult(result);
1314 
1315     } catch(Exception e) {
1316       throw new IOException("Using a row with null qualifier threw exception, should ");
1317     }
1318 
1319     // Null value
1320     try {
1321       Put put = new Put(ROW);
1322       put.add(FAMILY, QUALIFIER, null);
1323       ht.put(put);
1324 
1325       Get get = new Get(ROW);
1326       get.addColumn(FAMILY, QUALIFIER);
1327       Result result = ht.get(get);
1328       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1329 
1330       Scan scan = new Scan();
1331       scan.addColumn(FAMILY, QUALIFIER);
1332       result = getSingleScanResult(ht, scan);
1333       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1334 
1335       Delete delete = new Delete(ROW);
1336       delete.deleteColumns(FAMILY, QUALIFIER);
1337       ht.delete(delete);
1338 
1339       get = new Get(ROW);
1340       result = ht.get(get);
1341       assertEmptyResult(result);
1342 
1343     } catch(Exception e) {
1344       throw new IOException("Null values should be allowed, but threw exception");
1345     }
1346   }
1347 
1348   @Test
1349   public void testVersions() throws Exception {
1350     byte [] TABLE = Bytes.toBytes("testVersions");
1351 
1352     long [] STAMPS = makeStamps(20);
1353     byte [][] VALUES = makeNAscii(VALUE, 20);
1354 
1355     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
1356 
1357     // Insert 4 versions of same column
1358     Put put = new Put(ROW);
1359     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1360     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1361     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1362     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1363     ht.put(put);
1364 
1365     // Verify we can get each one properly
1366     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1367     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1368     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1369     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1370     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1371     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1372     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1373     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1374 
1375     // Verify we don't accidentally get others
1376     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1377     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1378     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1379     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1380     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1381     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1382 
1383     // Ensure maxVersions in query is respected
1384     Get get = new Get(ROW);
1385     get.addColumn(FAMILY, QUALIFIER);
1386     get.setMaxVersions(2);
1387     Result result = ht.get(get);
1388     assertNResult(result, ROW, FAMILY, QUALIFIER,
1389         new long [] {STAMPS[4], STAMPS[5]},
1390         new byte[][] {VALUES[4], VALUES[5]},
1391         0, 1);
1392 
1393     Scan scan = new Scan(ROW);
1394     scan.addColumn(FAMILY, QUALIFIER);
1395     scan.setMaxVersions(2);
1396     result = getSingleScanResult(ht, scan);
1397     assertNResult(result, ROW, FAMILY, QUALIFIER,
1398         new long [] {STAMPS[4], STAMPS[5]},
1399         new byte[][] {VALUES[4], VALUES[5]},
1400         0, 1);
1401 
1402     // Flush and redo
1403 
1404     TEST_UTIL.flush();
1405 
1406     // Verify we can get each one properly
1407     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1408     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1409     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1410     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1411     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1412     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1413     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1414     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1415 
1416     // Verify we don't accidentally get others
1417     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1418     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1419     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1420     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1421     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1422     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1423 
1424     // Ensure maxVersions in query is respected
1425     get = new Get(ROW);
1426     get.addColumn(FAMILY, QUALIFIER);
1427     get.setMaxVersions(2);
1428     result = ht.get(get);
1429     assertNResult(result, ROW, FAMILY, QUALIFIER,
1430         new long [] {STAMPS[4], STAMPS[5]},
1431         new byte[][] {VALUES[4], VALUES[5]},
1432         0, 1);
1433 
1434     scan = new Scan(ROW);
1435     scan.addColumn(FAMILY, QUALIFIER);
1436     scan.setMaxVersions(2);
1437     result = getSingleScanResult(ht, scan);
1438     assertNResult(result, ROW, FAMILY, QUALIFIER,
1439         new long [] {STAMPS[4], STAMPS[5]},
1440         new byte[][] {VALUES[4], VALUES[5]},
1441         0, 1);
1442 
1443 
1444     // Add some memstore and retest
1445 
1446     // Insert 4 more versions of same column and a dupe
1447     put = new Put(ROW);
1448     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1449     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1450     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1451     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1452     ht.put(put);
1453 
1454     // Ensure maxVersions in query is respected
1455     get = new Get(ROW);
1456     get.addColumn(FAMILY, QUALIFIER);
1457     get.setMaxVersions();
1458     result = ht.get(get);
1459     assertNResult(result, ROW, FAMILY, QUALIFIER,
1460         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1461         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1462         0, 7);
1463 
1464     scan = new Scan(ROW);
1465     scan.addColumn(FAMILY, QUALIFIER);
1466     scan.setMaxVersions();
1467     result = getSingleScanResult(ht, scan);
1468     assertNResult(result, ROW, FAMILY, QUALIFIER,
1469         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1470         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1471         0, 7);
1472 
1473     get = new Get(ROW);
1474     get.setMaxVersions();
1475     result = ht.get(get);
1476     assertNResult(result, ROW, FAMILY, QUALIFIER,
1477         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1478         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1479         0, 7);
1480 
1481     scan = new Scan(ROW);
1482     scan.setMaxVersions();
1483     result = getSingleScanResult(ht, scan);
1484     assertNResult(result, ROW, FAMILY, QUALIFIER,
1485         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1486         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1487         0, 7);
1488 
1489     // Verify we can get each one properly
1490     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1491     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1492     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1493     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1494     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1495     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1496     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1497     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1498 
1499     // Verify we don't accidentally get others
1500     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1501     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1502     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1503     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1504 
1505     // Ensure maxVersions of table is respected
1506 
1507     TEST_UTIL.flush();
1508 
1509     // Insert 4 more versions of same column and a dupe
1510     put = new Put(ROW);
1511     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1512     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1513     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1514     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1515     ht.put(put);
1516 
1517     get = new Get(ROW);
1518     get.addColumn(FAMILY, QUALIFIER);
1519     get.setMaxVersions(Integer.MAX_VALUE);
1520     result = ht.get(get);
1521     assertNResult(result, ROW, FAMILY, QUALIFIER,
1522         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1523         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1524         0, 9);
1525 
1526     scan = new Scan(ROW);
1527     scan.addColumn(FAMILY, QUALIFIER);
1528     scan.setMaxVersions(Integer.MAX_VALUE);
1529     result = getSingleScanResult(ht, scan);
1530     assertNResult(result, ROW, FAMILY, QUALIFIER,
1531         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1532         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1533         0, 9);
1534 
1535     // Delete a version in the memstore and a version in a storefile
1536     Delete delete = new Delete(ROW);
1537     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
1538     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
1539     ht.delete(delete);
1540 
1541     // Test that it's gone
1542     get = new Get(ROW);
1543     get.addColumn(FAMILY, QUALIFIER);
1544     get.setMaxVersions(Integer.MAX_VALUE);
1545     result = ht.get(get);
1546     assertNResult(result, ROW, FAMILY, QUALIFIER,
1547         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1548         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1549         0, 9);
1550 
1551     scan = new Scan(ROW);
1552     scan.addColumn(FAMILY, QUALIFIER);
1553     scan.setMaxVersions(Integer.MAX_VALUE);
1554     result = getSingleScanResult(ht, scan);
1555     assertNResult(result, ROW, FAMILY, QUALIFIER,
1556         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1557         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1558         0, 9);
1559 
1560   }
1561 
1562   @Test
1563   public void testVersionLimits() throws Exception {
1564     byte [] TABLE = Bytes.toBytes("testVersionLimits");
1565     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1566     int [] LIMITS = {1,3,5};
1567     long [] STAMPS = makeStamps(10);
1568     byte [][] VALUES = makeNAscii(VALUE, 10);
1569     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, LIMITS);
1570 
1571     // Insert limit + 1 on each family
1572     Put put = new Put(ROW);
1573     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1574     put.add(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1575     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1576     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1577     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1578     put.add(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1579     put.add(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1580     put.add(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1581     put.add(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1582     put.add(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1583     put.add(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1584     put.add(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1585     put.add(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1586     ht.put(put);
1587 
1588     // Verify we only get the right number out of each
1589 
1590     // Family0
1591 
1592     Get get = new Get(ROW);
1593     get.addColumn(FAMILIES[0], QUALIFIER);
1594     get.setMaxVersions(Integer.MAX_VALUE);
1595     Result result = ht.get(get);
1596     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1597         new long [] {STAMPS[1]},
1598         new byte[][] {VALUES[1]},
1599         0, 0);
1600 
1601     get = new Get(ROW);
1602     get.addFamily(FAMILIES[0]);
1603     get.setMaxVersions(Integer.MAX_VALUE);
1604     result = ht.get(get);
1605     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1606         new long [] {STAMPS[1]},
1607         new byte[][] {VALUES[1]},
1608         0, 0);
1609 
1610     Scan scan = new Scan(ROW);
1611     scan.addColumn(FAMILIES[0], QUALIFIER);
1612     scan.setMaxVersions(Integer.MAX_VALUE);
1613     result = getSingleScanResult(ht, scan);
1614     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1615         new long [] {STAMPS[1]},
1616         new byte[][] {VALUES[1]},
1617         0, 0);
1618 
1619     scan = new Scan(ROW);
1620     scan.addFamily(FAMILIES[0]);
1621     scan.setMaxVersions(Integer.MAX_VALUE);
1622     result = getSingleScanResult(ht, scan);
1623     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1624         new long [] {STAMPS[1]},
1625         new byte[][] {VALUES[1]},
1626         0, 0);
1627 
1628     // Family1
1629 
1630     get = new Get(ROW);
1631     get.addColumn(FAMILIES[1], QUALIFIER);
1632     get.setMaxVersions(Integer.MAX_VALUE);
1633     result = ht.get(get);
1634     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1635         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1636         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1637         0, 2);
1638 
1639     get = new Get(ROW);
1640     get.addFamily(FAMILIES[1]);
1641     get.setMaxVersions(Integer.MAX_VALUE);
1642     result = ht.get(get);
1643     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1644         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1645         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1646         0, 2);
1647 
1648     scan = new Scan(ROW);
1649     scan.addColumn(FAMILIES[1], QUALIFIER);
1650     scan.setMaxVersions(Integer.MAX_VALUE);
1651     result = getSingleScanResult(ht, scan);
1652     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1653         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1654         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1655         0, 2);
1656 
1657     scan = new Scan(ROW);
1658     scan.addFamily(FAMILIES[1]);
1659     scan.setMaxVersions(Integer.MAX_VALUE);
1660     result = getSingleScanResult(ht, scan);
1661     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1662         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1663         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1664         0, 2);
1665 
1666     // Family2
1667 
1668     get = new Get(ROW);
1669     get.addColumn(FAMILIES[2], QUALIFIER);
1670     get.setMaxVersions(Integer.MAX_VALUE);
1671     result = ht.get(get);
1672     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1673         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1674         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1675         0, 4);
1676 
1677     get = new Get(ROW);
1678     get.addFamily(FAMILIES[2]);
1679     get.setMaxVersions(Integer.MAX_VALUE);
1680     result = ht.get(get);
1681     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1682         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1683         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1684         0, 4);
1685 
1686     scan = new Scan(ROW);
1687     scan.addColumn(FAMILIES[2], QUALIFIER);
1688     scan.setMaxVersions(Integer.MAX_VALUE);
1689     result = getSingleScanResult(ht, scan);
1690     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1691         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1692         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1693         0, 4);
1694 
1695     scan = new Scan(ROW);
1696     scan.addFamily(FAMILIES[2]);
1697     scan.setMaxVersions(Integer.MAX_VALUE);
1698     result = getSingleScanResult(ht, scan);
1699     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1700         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1701         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1702         0, 4);
1703 
1704     // Try all families
1705 
1706     get = new Get(ROW);
1707     get.setMaxVersions(Integer.MAX_VALUE);
1708     result = ht.get(get);
1709     assertTrue("Expected 9 keys but received " + result.size(),
1710         result.size() == 9);
1711 
1712     get = new Get(ROW);
1713     get.addFamily(FAMILIES[0]);
1714     get.addFamily(FAMILIES[1]);
1715     get.addFamily(FAMILIES[2]);
1716     get.setMaxVersions(Integer.MAX_VALUE);
1717     result = ht.get(get);
1718     assertTrue("Expected 9 keys but received " + result.size(),
1719         result.size() == 9);
1720 
1721     get = new Get(ROW);
1722     get.addColumn(FAMILIES[0], QUALIFIER);
1723     get.addColumn(FAMILIES[1], QUALIFIER);
1724     get.addColumn(FAMILIES[2], QUALIFIER);
1725     get.setMaxVersions(Integer.MAX_VALUE);
1726     result = ht.get(get);
1727     assertTrue("Expected 9 keys but received " + result.size(),
1728         result.size() == 9);
1729 
1730     scan = new Scan(ROW);
1731     scan.setMaxVersions(Integer.MAX_VALUE);
1732     result = getSingleScanResult(ht, scan);
1733     assertTrue("Expected 9 keys but received " + result.size(),
1734         result.size() == 9);
1735 
1736     scan = new Scan(ROW);
1737     scan.setMaxVersions(Integer.MAX_VALUE);
1738     scan.addFamily(FAMILIES[0]);
1739     scan.addFamily(FAMILIES[1]);
1740     scan.addFamily(FAMILIES[2]);
1741     result = getSingleScanResult(ht, scan);
1742     assertTrue("Expected 9 keys but received " + result.size(),
1743         result.size() == 9);
1744 
1745     scan = new Scan(ROW);
1746     scan.setMaxVersions(Integer.MAX_VALUE);
1747     scan.addColumn(FAMILIES[0], QUALIFIER);
1748     scan.addColumn(FAMILIES[1], QUALIFIER);
1749     scan.addColumn(FAMILIES[2], QUALIFIER);
1750     result = getSingleScanResult(ht, scan);
1751     assertTrue("Expected 9 keys but received " + result.size(),
1752         result.size() == 9);
1753 
1754   }
1755 
1756   @Test
1757   public void testDeleteFamilyVersion() throws Exception {
1758     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1759     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersion");
1760 
1761     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
1762     byte [][] VALUES = makeN(VALUE, 5);
1763     long [] ts = {1000, 2000, 3000, 4000, 5000};
1764 
1765     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1766 
1767     Put put = new Put(ROW);
1768     for (int q = 0; q < 1; q++)
1769       for (int t = 0; t < 5; t++)
1770         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1771     ht.put(put);
1772     admin.flush(TABLE);
1773 
1774     Delete delete = new Delete(ROW);
1775     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1776     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1777     ht.delete(delete);
1778     admin.flush(TABLE);
1779 
1780     for (int i = 0; i < 1; i++) {
1781       Get get = new Get(ROW);
1782       get.addColumn(FAMILY, QUALIFIERS[i]);
1783       get.setMaxVersions(Integer.MAX_VALUE);
1784       Result result = ht.get(get);
1785       // verify version '1000'/'3000'/'5000' remains for all columns
1786       assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
1787           new long [] {ts[0], ts[2], ts[4]},
1788           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1789           0, 2);
1790     }
1791     ht.close();
1792     admin.close();
1793   }
1794 
1795   @Test
1796   public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
1797     byte [] TABLE = Bytes.toBytes("testDeleteFamilyVersionWithOtherDeletes");
1798 
1799     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
1800     byte [][] VALUES = makeN(VALUE, 5);
1801     long [] ts = {1000, 2000, 3000, 4000, 5000};
1802 
1803     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
1804     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 5);
1805     Put put = null;
1806     Result result = null;
1807     Get get = null;
1808     Delete delete = null;
1809 
1810     // 1. put on ROW
1811     put = new Put(ROW);
1812     for (int q = 0; q < 5; q++)
1813       for (int t = 0; t < 5; t++)
1814         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1815     ht.put(put);
1816     admin.flush(TABLE);
1817 
1818     // 2. put on ROWS[0]
1819     byte [] ROW2 = Bytes.toBytes("myRowForTest");
1820     put = new Put(ROW2);
1821     for (int q = 0; q < 5; q++)
1822       for (int t = 0; t < 5; t++)
1823         put.add(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1824     ht.put(put);
1825     admin.flush(TABLE);
1826 
1827     // 3. delete on ROW
1828     delete = new Delete(ROW);
1829     // delete version <= 2000 of all columns
1830     // note: deleteFamily must be the first since it will mask
1831     // the subsequent other type deletes!
1832     delete.deleteFamily(FAMILY, ts[1]);
1833     // delete version '4000' of all columns
1834     delete.deleteFamilyVersion(FAMILY, ts[3]);
1835    // delete version <= 3000 of column 0
1836     delete.deleteColumns(FAMILY, QUALIFIERS[0], ts[2]);
1837     // delete version <= 5000 of column 2
1838     delete.deleteColumns(FAMILY, QUALIFIERS[2], ts[4]);
1839     // delete version 5000 of column 4
1840     delete.deleteColumn(FAMILY, QUALIFIERS[4], ts[4]);
1841     ht.delete(delete);
1842     admin.flush(TABLE);
1843 
1844      // 4. delete on ROWS[0]
1845     delete = new Delete(ROW2);
1846     delete.deleteFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1847     delete.deleteFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1848     ht.delete(delete);
1849     admin.flush(TABLE);
1850 
1851     // 5. check ROW
1852     get = new Get(ROW);
1853     get.addColumn(FAMILY, QUALIFIERS[0]);
1854     get.setMaxVersions(Integer.MAX_VALUE);
1855     result = ht.get(get);
1856     assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
1857         new long [] {ts[4]},
1858         new byte[][] {VALUES[4]},
1859         0, 0);
1860 
1861     get = new Get(ROW);
1862     get.addColumn(FAMILY, QUALIFIERS[1]);
1863     get.setMaxVersions(Integer.MAX_VALUE);
1864     result = ht.get(get);
1865     assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
1866         new long [] {ts[2], ts[4]},
1867         new byte[][] {VALUES[2], VALUES[4]},
1868         0, 1);
1869 
1870     get = new Get(ROW);
1871     get.addColumn(FAMILY, QUALIFIERS[2]);
1872     get.setMaxVersions(Integer.MAX_VALUE);
1873     result = ht.get(get);
1874     assertEquals(0, result.size());
1875 
1876     get = new Get(ROW);
1877     get.addColumn(FAMILY, QUALIFIERS[3]);
1878     get.setMaxVersions(Integer.MAX_VALUE);
1879     result = ht.get(get);
1880     assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
1881         new long [] {ts[2], ts[4]},
1882         new byte[][] {VALUES[2], VALUES[4]},
1883         0, 1);
1884 
1885     get = new Get(ROW);
1886     get.addColumn(FAMILY, QUALIFIERS[4]);
1887     get.setMaxVersions(Integer.MAX_VALUE);
1888     result = ht.get(get);
1889     assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
1890         new long [] {ts[2]},
1891         new byte[][] {VALUES[2]},
1892         0, 0);
1893 
1894     // 6. check ROWS[0]
1895     for (int i = 0; i < 5; i++) {
1896       get = new Get(ROW2);
1897       get.addColumn(FAMILY, QUALIFIERS[i]);
1898       get.setMaxVersions(Integer.MAX_VALUE);
1899       result = ht.get(get);
1900       // verify version '1000'/'3000'/'5000' remains for all columns
1901       assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
1902           new long [] {ts[0], ts[2], ts[4]},
1903           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1904           0, 2);
1905     }
1906     ht.close();
1907     admin.close();
1908   }
1909 
1910   @Test
1911   public void testDeletes() throws Exception {
1912     byte [] TABLE = Bytes.toBytes("testDeletes");
1913 
1914     byte [][] ROWS = makeNAscii(ROW, 6);
1915     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1916     byte [][] VALUES = makeN(VALUE, 5);
1917     long [] ts = {1000, 2000, 3000, 4000, 5000};
1918 
1919     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, 3);
1920 
1921     Put put = new Put(ROW);
1922     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1923     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
1924     ht.put(put);
1925 
1926     Delete delete = new Delete(ROW);
1927     delete.deleteFamily(FAMILIES[0], ts[0]);
1928     ht.delete(delete);
1929 
1930     Get get = new Get(ROW);
1931     get.addFamily(FAMILIES[0]);
1932     get.setMaxVersions(Integer.MAX_VALUE);
1933     Result result = ht.get(get);
1934     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1935         new long [] {ts[1]},
1936         new byte[][] {VALUES[1]},
1937         0, 0);
1938 
1939     Scan scan = new Scan(ROW);
1940     scan.addFamily(FAMILIES[0]);
1941     scan.setMaxVersions(Integer.MAX_VALUE);
1942     result = getSingleScanResult(ht, scan);
1943     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1944         new long [] {ts[1]},
1945         new byte[][] {VALUES[1]},
1946         0, 0);
1947 
1948     // Test delete latest version
1949     put = new Put(ROW);
1950     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1951     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
1952     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
1953     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
1954     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
1955     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
1956     ht.put(put);
1957 
1958     delete = new Delete(ROW);
1959     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
1960     ht.delete(delete);
1961 
1962     get = new Get(ROW);
1963     get.addColumn(FAMILIES[0], QUALIFIER);
1964     get.setMaxVersions(Integer.MAX_VALUE);
1965     result = ht.get(get);
1966     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1967         new long [] {ts[1], ts[2], ts[3]},
1968         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1969         0, 2);
1970 
1971     scan = new Scan(ROW);
1972     scan.addColumn(FAMILIES[0], QUALIFIER);
1973     scan.setMaxVersions(Integer.MAX_VALUE);
1974     result = getSingleScanResult(ht, scan);
1975     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1976         new long [] {ts[1], ts[2], ts[3]},
1977         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1978         0, 2);
1979 
1980     // Test for HBASE-1847
1981     delete = new Delete(ROW);
1982     delete.deleteColumn(FAMILIES[0], null);
1983     ht.delete(delete);
1984 
1985     // Cleanup null qualifier
1986     delete = new Delete(ROW);
1987     delete.deleteColumns(FAMILIES[0], null);
1988     ht.delete(delete);
1989 
1990     // Expected client behavior might be that you can re-put deleted values
1991     // But alas, this is not to be.  We can't put them back in either case.
1992 
1993     put = new Put(ROW);
1994     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
1995     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
1996     ht.put(put);
1997 
1998 
1999     // It used to be due to the internal implementation of Get, that
2000     // the Get() call would return ts[4] UNLIKE the Scan below. With
2001     // the switch to using Scan for Get this is no longer the case.
2002     get = new Get(ROW);
2003     get.addFamily(FAMILIES[0]);
2004     get.setMaxVersions(Integer.MAX_VALUE);
2005     result = ht.get(get);
2006     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2007         new long [] {ts[1], ts[2], ts[3]},
2008         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2009         0, 2);
2010 
2011     // The Scanner returns the previous values, the expected-naive-unexpected behavior
2012 
2013     scan = new Scan(ROW);
2014     scan.addFamily(FAMILIES[0]);
2015     scan.setMaxVersions(Integer.MAX_VALUE);
2016     result = getSingleScanResult(ht, scan);
2017     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2018         new long [] {ts[1], ts[2], ts[3]},
2019         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2020         0, 2);
2021 
2022     // Test deleting an entire family from one row but not the other various ways
2023 
2024     put = new Put(ROWS[0]);
2025     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2026     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2027     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2028     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2029     ht.put(put);
2030 
2031     put = new Put(ROWS[1]);
2032     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2033     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2034     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2035     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2036     ht.put(put);
2037 
2038     put = new Put(ROWS[2]);
2039     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2040     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2041     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2042     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2043     ht.put(put);
2044 
2045     // Assert that above went in.
2046     get = new Get(ROWS[2]);
2047     get.addFamily(FAMILIES[1]);
2048     get.addFamily(FAMILIES[2]);
2049     get.setMaxVersions(Integer.MAX_VALUE);
2050     result = ht.get(get);
2051     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
2052         result.size() == 4);
2053 
2054     delete = new Delete(ROWS[0]);
2055     delete.deleteFamily(FAMILIES[2]);
2056     ht.delete(delete);
2057 
2058     delete = new Delete(ROWS[1]);
2059     delete.deleteColumns(FAMILIES[1], QUALIFIER);
2060     ht.delete(delete);
2061 
2062     delete = new Delete(ROWS[2]);
2063     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2064     delete.deleteColumn(FAMILIES[1], QUALIFIER);
2065     delete.deleteColumn(FAMILIES[2], QUALIFIER);
2066     ht.delete(delete);
2067 
2068     get = new Get(ROWS[0]);
2069     get.addFamily(FAMILIES[1]);
2070     get.addFamily(FAMILIES[2]);
2071     get.setMaxVersions(Integer.MAX_VALUE);
2072     result = ht.get(get);
2073     assertTrue("Expected 2 keys but received " + result.size(),
2074         result.size() == 2);
2075     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2076         new long [] {ts[0], ts[1]},
2077         new byte[][] {VALUES[0], VALUES[1]},
2078         0, 1);
2079 
2080     scan = new Scan(ROWS[0]);
2081     scan.addFamily(FAMILIES[1]);
2082     scan.addFamily(FAMILIES[2]);
2083     scan.setMaxVersions(Integer.MAX_VALUE);
2084     result = getSingleScanResult(ht, scan);
2085     assertTrue("Expected 2 keys but received " + result.size(),
2086         result.size() == 2);
2087     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2088         new long [] {ts[0], ts[1]},
2089         new byte[][] {VALUES[0], VALUES[1]},
2090         0, 1);
2091 
2092     get = new Get(ROWS[1]);
2093     get.addFamily(FAMILIES[1]);
2094     get.addFamily(FAMILIES[2]);
2095     get.setMaxVersions(Integer.MAX_VALUE);
2096     result = ht.get(get);
2097     assertTrue("Expected 2 keys but received " + result.size(),
2098         result.size() == 2);
2099 
2100     scan = new Scan(ROWS[1]);
2101     scan.addFamily(FAMILIES[1]);
2102     scan.addFamily(FAMILIES[2]);
2103     scan.setMaxVersions(Integer.MAX_VALUE);
2104     result = getSingleScanResult(ht, scan);
2105     assertTrue("Expected 2 keys but received " + result.size(),
2106         result.size() == 2);
2107 
2108     get = new Get(ROWS[2]);
2109     get.addFamily(FAMILIES[1]);
2110     get.addFamily(FAMILIES[2]);
2111     get.setMaxVersions(Integer.MAX_VALUE);
2112     result = ht.get(get);
2113     assertEquals(1, result.size());
2114     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2115         new long [] {ts[2]},
2116         new byte[][] {VALUES[2]},
2117         0, 0);
2118 
2119     scan = new Scan(ROWS[2]);
2120     scan.addFamily(FAMILIES[1]);
2121     scan.addFamily(FAMILIES[2]);
2122     scan.setMaxVersions(Integer.MAX_VALUE);
2123     result = getSingleScanResult(ht, scan);
2124     assertEquals(1, result.size());
2125     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2126         new long [] {ts[2]},
2127         new byte[][] {VALUES[2]},
2128         0, 0);
2129 
2130     // Test if we delete the family first in one row (HBASE-1541)
2131 
2132     delete = new Delete(ROWS[3]);
2133     delete.deleteFamily(FAMILIES[1]);
2134     ht.delete(delete);
2135 
2136     put = new Put(ROWS[3]);
2137     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
2138     ht.put(put);
2139 
2140     put = new Put(ROWS[4]);
2141     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
2142     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
2143     ht.put(put);
2144 
2145     get = new Get(ROWS[3]);
2146     get.addFamily(FAMILIES[1]);
2147     get.addFamily(FAMILIES[2]);
2148     get.setMaxVersions(Integer.MAX_VALUE);
2149     result = ht.get(get);
2150     assertTrue("Expected 1 key but received " + result.size(),
2151         result.size() == 1);
2152 
2153     get = new Get(ROWS[4]);
2154     get.addFamily(FAMILIES[1]);
2155     get.addFamily(FAMILIES[2]);
2156     get.setMaxVersions(Integer.MAX_VALUE);
2157     result = ht.get(get);
2158     assertTrue("Expected 2 keys but received " + result.size(),
2159         result.size() == 2);
2160 
2161     scan = new Scan(ROWS[3]);
2162     scan.addFamily(FAMILIES[1]);
2163     scan.addFamily(FAMILIES[2]);
2164     scan.setMaxVersions(Integer.MAX_VALUE);
2165     ResultScanner scanner = ht.getScanner(scan);
2166     result = scanner.next();
2167     assertTrue("Expected 1 key but received " + result.size(),
2168         result.size() == 1);
2169     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2170     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2171     result = scanner.next();
2172     assertTrue("Expected 2 keys but received " + result.size(),
2173         result.size() == 2);
2174     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2175     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2176     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2177     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2178     scanner.close();
2179 
2180     // Add test of bulk deleting.
2181     for (int i = 0; i < 10; i++) {
2182       byte [] bytes = Bytes.toBytes(i);
2183       put = new Put(bytes);
2184       put.setDurability(Durability.SKIP_WAL);
2185       put.add(FAMILIES[0], QUALIFIER, bytes);
2186       ht.put(put);
2187     }
2188     for (int i = 0; i < 10; i++) {
2189       byte [] bytes = Bytes.toBytes(i);
2190       get = new Get(bytes);
2191       get.addFamily(FAMILIES[0]);
2192       result = ht.get(get);
2193       assertTrue(result.size() == 1);
2194     }
2195     ArrayList<Delete> deletes = new ArrayList<Delete>();
2196     for (int i = 0; i < 10; i++) {
2197       byte [] bytes = Bytes.toBytes(i);
2198       delete = new Delete(bytes);
2199       delete.deleteFamily(FAMILIES[0]);
2200       deletes.add(delete);
2201     }
2202     ht.delete(deletes);
2203     for (int i = 0; i < 10; i++) {
2204       byte [] bytes = Bytes.toBytes(i);
2205       get = new Get(bytes);
2206       get.addFamily(FAMILIES[0]);
2207       result = ht.get(get);
2208       assertTrue(result.size() == 0);
2209     }
2210   }
2211 
2212   /*
2213    * Baseline "scalability" test.
2214    *
2215    * Tests one hundred families, one million columns, one million versions
2216    */
2217   @Ignore @Test
2218   public void testMillions() throws Exception {
2219 
2220     // 100 families
2221 
2222     // millions of columns
2223 
2224     // millions of versions
2225 
2226   }
2227 
2228   @Ignore @Test
2229   public void testMultipleRegionsAndBatchPuts() throws Exception {
2230     // Two family table
2231 
2232     // Insert lots of rows
2233 
2234     // Insert to the same row with batched puts
2235 
2236     // Insert to multiple rows with batched puts
2237 
2238     // Split the table
2239 
2240     // Get row from first region
2241 
2242     // Get row from second region
2243 
2244     // Scan all rows
2245 
2246     // Insert to multiple regions with batched puts
2247 
2248     // Get row from first region
2249 
2250     // Get row from second region
2251 
2252     // Scan all rows
2253 
2254 
2255   }
2256 
2257   @Ignore @Test
2258   public void testMultipleRowMultipleFamily() throws Exception {
2259 
2260   }
2261 
2262   //
2263   // JIRA Testers
2264   //
2265 
2266   /**
2267    * HBASE-867
2268    *    If millions of columns in a column family, hbase scanner won't come up
2269    *
2270    *    Test will create numRows rows, each with numColsPerRow columns
2271    *    (1 version each), and attempt to scan them all.
2272    *
2273    *    To test at scale, up numColsPerRow to the millions
2274    *    (have not gotten that to work running as junit though)
2275    */
2276   @Test
2277   public void testJiraTest867() throws Exception {
2278     int numRows = 10;
2279     int numColsPerRow = 2000;
2280 
2281     byte [] TABLE = Bytes.toBytes("testJiraTest867");
2282 
2283     byte [][] ROWS = makeN(ROW, numRows);
2284     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2285 
2286     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
2287 
2288     // Insert rows
2289 
2290     for(int i=0;i<numRows;i++) {
2291       Put put = new Put(ROWS[i]);
2292       put.setDurability(Durability.SKIP_WAL);
2293       for(int j=0;j<numColsPerRow;j++) {
2294         put.add(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2295       }
2296       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2297           "only contains " + put.size(), put.size() == numColsPerRow);
2298       ht.put(put);
2299     }
2300 
2301     // Get a row
2302     Get get = new Get(ROWS[numRows-1]);
2303     Result result = ht.get(get);
2304     assertNumKeys(result, numColsPerRow);
2305     Cell [] keys = result.rawCells();
2306     for(int i=0;i<result.size();i++) {
2307       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2308     }
2309 
2310     // Scan the rows
2311     Scan scan = new Scan();
2312     ResultScanner scanner = ht.getScanner(scan);
2313     int rowCount = 0;
2314     while((result = scanner.next()) != null) {
2315       assertNumKeys(result, numColsPerRow);
2316       Cell [] kvs = result.rawCells();
2317       for(int i=0;i<numColsPerRow;i++) {
2318         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2319       }
2320       rowCount++;
2321     }
2322     scanner.close();
2323     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2324         + rowCount + " rows", rowCount == numRows);
2325 
2326     // flush and try again
2327 
2328     TEST_UTIL.flush();
2329 
2330     // Get a row
2331     get = new Get(ROWS[numRows-1]);
2332     result = ht.get(get);
2333     assertNumKeys(result, numColsPerRow);
2334     keys = result.rawCells();
2335     for(int i=0;i<result.size();i++) {
2336       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2337     }
2338 
2339     // Scan the rows
2340     scan = new Scan();
2341     scanner = ht.getScanner(scan);
2342     rowCount = 0;
2343     while((result = scanner.next()) != null) {
2344       assertNumKeys(result, numColsPerRow);
2345       Cell [] kvs = result.rawCells();
2346       for(int i=0;i<numColsPerRow;i++) {
2347         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2348       }
2349       rowCount++;
2350     }
2351     scanner.close();
2352     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2353         + rowCount + " rows", rowCount == numRows);
2354 
2355   }
2356 
2357   /**
2358    * HBASE-861
2359    *    get with timestamp will return a value if there is a version with an
2360    *    earlier timestamp
2361    */
2362   @Test
2363   public void testJiraTest861() throws Exception {
2364 
2365     byte [] TABLE = Bytes.toBytes("testJiraTest861");
2366     byte [][] VALUES = makeNAscii(VALUE, 7);
2367     long [] STAMPS = makeStamps(7);
2368 
2369     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2370 
2371     // Insert three versions
2372 
2373     Put put = new Put(ROW);
2374     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2375     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2376     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2377     ht.put(put);
2378 
2379     // Get the middle value
2380     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2381 
2382     // Try to get one version before (expect fail)
2383     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2384 
2385     // Try to get one version after (expect fail)
2386     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2387 
2388     // Try same from storefile
2389     TEST_UTIL.flush();
2390     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2391     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2392     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2393 
2394     // Insert two more versions surrounding others, into memstore
2395     put = new Put(ROW);
2396     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2397     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2398     ht.put(put);
2399 
2400     // Check we can get everything we should and can't get what we shouldn't
2401     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2402     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2403     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2404     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2405     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2406     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2407     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2408 
2409     // Try same from two storefiles
2410     TEST_UTIL.flush();
2411     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2412     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2413     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2414     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2415     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2416     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2417     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2418 
2419   }
2420 
2421   /**
2422    * HBASE-33
2423    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2424    *    particular column and row between two timestamps
2425    */
2426   @Test
2427   public void testJiraTest33() throws Exception {
2428 
2429     byte [] TABLE = Bytes.toBytes("testJiraTest33");
2430     byte [][] VALUES = makeNAscii(VALUE, 7);
2431     long [] STAMPS = makeStamps(7);
2432 
2433     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2434 
2435     // Insert lots versions
2436 
2437     Put put = new Put(ROW);
2438     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2439     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2440     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2441     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2442     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2443     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2444     ht.put(put);
2445 
2446     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2447     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2448     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2449     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2450 
2451     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2452     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2453     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2454     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2455 
2456     // Try same from storefile
2457     TEST_UTIL.flush();
2458 
2459     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2460     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2461     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2462     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2463 
2464     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2465     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2466     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2467     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2468 
2469   }
2470 
2471   /**
2472    * HBASE-1014
2473    *    commit(BatchUpdate) method should return timestamp
2474    */
2475   @Test
2476   public void testJiraTest1014() throws Exception {
2477 
2478     byte [] TABLE = Bytes.toBytes("testJiraTest1014");
2479 
2480     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2481 
2482     long manualStamp = 12345;
2483 
2484     // Insert lots versions
2485 
2486     Put put = new Put(ROW);
2487     put.add(FAMILY, QUALIFIER, manualStamp, VALUE);
2488     ht.put(put);
2489 
2490     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2491     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2492     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2493 
2494   }
2495 
2496   /**
2497    * HBASE-1182
2498    *    Scan for columns > some timestamp
2499    */
2500   @Test
2501   public void testJiraTest1182() throws Exception {
2502 
2503     byte [] TABLE = Bytes.toBytes("testJiraTest1182");
2504     byte [][] VALUES = makeNAscii(VALUE, 7);
2505     long [] STAMPS = makeStamps(7);
2506 
2507     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2508 
2509     // Insert lots versions
2510 
2511     Put put = new Put(ROW);
2512     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2513     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2514     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2515     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2516     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2517     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2518     ht.put(put);
2519 
2520     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2521     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2522     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2523 
2524     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2525     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2526     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2527 
2528     // Try same from storefile
2529     TEST_UTIL.flush();
2530 
2531     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2532     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2533     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2534 
2535     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2536     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2537     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2538   }
2539 
2540   /**
2541    * HBASE-52
2542    *    Add a means of scanning over all versions
2543    */
2544   @Test
2545   public void testJiraTest52() throws Exception {
2546     byte [] TABLE = Bytes.toBytes("testJiraTest52");
2547     byte [][] VALUES = makeNAscii(VALUE, 7);
2548     long [] STAMPS = makeStamps(7);
2549 
2550     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
2551 
2552     // Insert lots versions
2553 
2554     Put put = new Put(ROW);
2555     put.add(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2556     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2557     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2558     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2559     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2560     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2561     ht.put(put);
2562 
2563     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2564 
2565     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2566 
2567     // Try same from storefile
2568     TEST_UTIL.flush();
2569 
2570     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2571 
2572     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2573   }
2574 
2575   //
2576   // Bulk Testers
2577   //
2578 
2579   private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2580       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2581       int start, int end)
2582   throws IOException {
2583     Get get = new Get(row);
2584     get.addColumn(family, qualifier);
2585     get.setMaxVersions(Integer.MAX_VALUE);
2586     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2587     Result result = ht.get(get);
2588     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2589   }
2590 
2591   private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2592       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2593   throws IOException {
2594     Get get = new Get(row);
2595     get.addColumn(family, qualifier);
2596     get.setMaxVersions(Integer.MAX_VALUE);
2597     get.setTimeRange(stamps[start], stamps[end]+1);
2598     Result result = ht.get(get);
2599     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2600   }
2601 
2602   private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2603       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2604   throws IOException {
2605     Get get = new Get(row);
2606     get.addColumn(family, qualifier);
2607     get.setMaxVersions(Integer.MAX_VALUE);
2608     Result result = ht.get(get);
2609     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2610   }
2611 
2612   private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2613       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2614       int start, int end)
2615   throws IOException {
2616     Scan scan = new Scan(row);
2617     scan.addColumn(family, qualifier);
2618     scan.setMaxVersions(Integer.MAX_VALUE);
2619     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2620     Result result = getSingleScanResult(ht, scan);
2621     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2622   }
2623 
2624   private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2625       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2626   throws IOException {
2627     Scan scan = new Scan(row);
2628     scan.addColumn(family, qualifier);
2629     scan.setMaxVersions(Integer.MAX_VALUE);
2630     scan.setTimeRange(stamps[start], stamps[end]+1);
2631     Result result = getSingleScanResult(ht, scan);
2632     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2633   }
2634 
2635   private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2636       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2637   throws IOException {
2638     Scan scan = new Scan(row);
2639     scan.addColumn(family, qualifier);
2640     scan.setMaxVersions(Integer.MAX_VALUE);
2641     Result result = getSingleScanResult(ht, scan);
2642     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2643   }
2644 
2645   private void getVersionAndVerify(Table ht, byte [] row, byte [] family,
2646       byte [] qualifier, long stamp, byte [] value)
2647   throws Exception {
2648     Get get = new Get(row);
2649     get.addColumn(family, qualifier);
2650     get.setTimeStamp(stamp);
2651     get.setMaxVersions(Integer.MAX_VALUE);
2652     Result result = ht.get(get);
2653     assertSingleResult(result, row, family, qualifier, stamp, value);
2654   }
2655 
2656   private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family,
2657       byte [] qualifier, long stamp)
2658   throws Exception {
2659     Get get = new Get(row);
2660     get.addColumn(family, qualifier);
2661     get.setTimeStamp(stamp);
2662     get.setMaxVersions(Integer.MAX_VALUE);
2663     Result result = ht.get(get);
2664     assertEmptyResult(result);
2665   }
2666 
2667   private void scanVersionAndVerify(Table ht, byte [] row, byte [] family,
2668       byte [] qualifier, long stamp, byte [] value)
2669   throws Exception {
2670     Scan scan = new Scan(row);
2671     scan.addColumn(family, qualifier);
2672     scan.setTimeStamp(stamp);
2673     scan.setMaxVersions(Integer.MAX_VALUE);
2674     Result result = getSingleScanResult(ht, scan);
2675     assertSingleResult(result, row, family, qualifier, stamp, value);
2676   }
2677 
2678   private void scanVersionAndVerifyMissing(Table ht, byte [] row,
2679       byte [] family, byte [] qualifier, long stamp)
2680   throws Exception {
2681     Scan scan = new Scan(row);
2682     scan.addColumn(family, qualifier);
2683     scan.setTimeStamp(stamp);
2684     scan.setMaxVersions(Integer.MAX_VALUE);
2685     Result result = getSingleScanResult(ht, scan);
2686     assertNullResult(result);
2687   }
2688 
2689   private void getTestNull(Table ht, byte [] row, byte [] family,
2690       byte [] value)
2691   throws Exception {
2692 
2693     Get get = new Get(row);
2694     get.addColumn(family, null);
2695     Result result = ht.get(get);
2696     assertSingleResult(result, row, family, null, value);
2697 
2698     get = new Get(row);
2699     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2700     result = ht.get(get);
2701     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2702 
2703     get = new Get(row);
2704     get.addFamily(family);
2705     result = ht.get(get);
2706     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2707 
2708     get = new Get(row);
2709     result = ht.get(get);
2710     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2711 
2712   }
2713 
2714   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value)
2715       throws Exception {
2716     scanTestNull(ht, row, family, value, false);
2717   }
2718 
2719   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value,
2720       boolean isReversedScan) throws Exception {
2721 
2722     Scan scan = new Scan();
2723     scan.setReversed(isReversedScan);
2724     scan.addColumn(family, null);
2725     Result result = getSingleScanResult(ht, scan);
2726     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2727 
2728     scan = new Scan();
2729     scan.setReversed(isReversedScan);
2730     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2731     result = getSingleScanResult(ht, scan);
2732     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2733 
2734     scan = new Scan();
2735     scan.setReversed(isReversedScan);
2736     scan.addFamily(family);
2737     result = getSingleScanResult(ht, scan);
2738     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2739 
2740     scan = new Scan();
2741     scan.setReversed(isReversedScan);
2742     result = getSingleScanResult(ht, scan);
2743     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2744 
2745   }
2746 
2747   private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2748       byte [][] QUALIFIERS, byte [][] VALUES)
2749   throws Exception {
2750 
2751     // Single column from memstore
2752     Get get = new Get(ROWS[0]);
2753     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2754     Result result = ht.get(get);
2755     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2756 
2757     // Single column from storefile
2758     get = new Get(ROWS[0]);
2759     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2760     result = ht.get(get);
2761     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2762 
2763     // Single column from storefile, family match
2764     get = new Get(ROWS[0]);
2765     get.addFamily(FAMILIES[7]);
2766     result = ht.get(get);
2767     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2768 
2769     // Two columns, one from memstore one from storefile, same family,
2770     // wildcard match
2771     get = new Get(ROWS[0]);
2772     get.addFamily(FAMILIES[4]);
2773     result = ht.get(get);
2774     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2775         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2776 
2777     // Two columns, one from memstore one from storefile, same family,
2778     // explicit match
2779     get = new Get(ROWS[0]);
2780     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2781     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2782     result = ht.get(get);
2783     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2784         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2785 
2786     // Three column, one from memstore two from storefile, different families,
2787     // wildcard match
2788     get = new Get(ROWS[0]);
2789     get.addFamily(FAMILIES[4]);
2790     get.addFamily(FAMILIES[7]);
2791     result = ht.get(get);
2792     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2793         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2794 
2795     // Multiple columns from everywhere storefile, many family, wildcard
2796     get = new Get(ROWS[0]);
2797     get.addFamily(FAMILIES[2]);
2798     get.addFamily(FAMILIES[4]);
2799     get.addFamily(FAMILIES[6]);
2800     get.addFamily(FAMILIES[7]);
2801     result = ht.get(get);
2802     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2803         new int [][] {
2804           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2805     });
2806 
2807     // Multiple columns from everywhere storefile, many family, wildcard
2808     get = new Get(ROWS[0]);
2809     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2810     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2811     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2812     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2813     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2814     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2815     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2816     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2817     result = ht.get(get);
2818     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2819         new int [][] {
2820           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2821     });
2822 
2823     // Everything
2824     get = new Get(ROWS[0]);
2825     result = ht.get(get);
2826     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2827         new int [][] {
2828           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2829     });
2830 
2831     // Get around inserted columns
2832 
2833     get = new Get(ROWS[1]);
2834     result = ht.get(get);
2835     assertEmptyResult(result);
2836 
2837     get = new Get(ROWS[0]);
2838     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
2839     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
2840     result = ht.get(get);
2841     assertEmptyResult(result);
2842 
2843   }
2844 
2845   private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2846       byte [][] QUALIFIERS, byte [][] VALUES)
2847   throws Exception {
2848 
2849     // Single column from memstore
2850     Scan scan = new Scan();
2851     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2852     Result result = getSingleScanResult(ht, scan);
2853     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2854 
2855     // Single column from storefile
2856     scan = new Scan();
2857     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2858     result = getSingleScanResult(ht, scan);
2859     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2860 
2861     // Single column from storefile, family match
2862     scan = new Scan();
2863     scan.addFamily(FAMILIES[7]);
2864     result = getSingleScanResult(ht, scan);
2865     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2866 
2867     // Two columns, one from memstore one from storefile, same family,
2868     // wildcard match
2869     scan = new Scan();
2870     scan.addFamily(FAMILIES[4]);
2871     result = getSingleScanResult(ht, scan);
2872     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2873         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2874 
2875     // Two columns, one from memstore one from storefile, same family,
2876     // explicit match
2877     scan = new Scan();
2878     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2879     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2880     result = getSingleScanResult(ht, scan);
2881     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2882         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2883 
2884     // Three column, one from memstore two from storefile, different families,
2885     // wildcard match
2886     scan = new Scan();
2887     scan.addFamily(FAMILIES[4]);
2888     scan.addFamily(FAMILIES[7]);
2889     result = getSingleScanResult(ht, scan);
2890     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2891         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2892 
2893     // Multiple columns from everywhere storefile, many family, wildcard
2894     scan = new Scan();
2895     scan.addFamily(FAMILIES[2]);
2896     scan.addFamily(FAMILIES[4]);
2897     scan.addFamily(FAMILIES[6]);
2898     scan.addFamily(FAMILIES[7]);
2899     result = getSingleScanResult(ht, scan);
2900     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2901         new int [][] {
2902           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2903     });
2904 
2905     // Multiple columns from everywhere storefile, many family, wildcard
2906     scan = new Scan();
2907     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2908     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
2909     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2910     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
2911     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
2912     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
2913     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
2914     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
2915     result = getSingleScanResult(ht, scan);
2916     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2917         new int [][] {
2918           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2919     });
2920 
2921     // Everything
2922     scan = new Scan();
2923     result = getSingleScanResult(ht, scan);
2924     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2925         new int [][] {
2926           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2927     });
2928 
2929     // Scan around inserted columns
2930 
2931     scan = new Scan(ROWS[1]);
2932     result = getSingleScanResult(ht, scan);
2933     assertNullResult(result);
2934 
2935     scan = new Scan();
2936     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
2937     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
2938     result = getSingleScanResult(ht, scan);
2939     assertNullResult(result);
2940   }
2941 
2942   /**
2943    * Verify a single column using gets.
2944    * Expects family and qualifier arrays to be valid for at least
2945    * the range:  idx-2 < idx < idx+2
2946    */
2947   private void getVerifySingleColumn(Table ht,
2948       byte [][] ROWS, int ROWIDX,
2949       byte [][] FAMILIES, int FAMILYIDX,
2950       byte [][] QUALIFIERS, int QUALIFIERIDX,
2951       byte [][] VALUES, int VALUEIDX)
2952   throws Exception {
2953 
2954     Get get = new Get(ROWS[ROWIDX]);
2955     Result result = ht.get(get);
2956     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2957         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2958 
2959     get = new Get(ROWS[ROWIDX]);
2960     get.addFamily(FAMILIES[FAMILYIDX]);
2961     result = ht.get(get);
2962     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2963         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2964 
2965     get = new Get(ROWS[ROWIDX]);
2966     get.addFamily(FAMILIES[FAMILYIDX-2]);
2967     get.addFamily(FAMILIES[FAMILYIDX]);
2968     get.addFamily(FAMILIES[FAMILYIDX+2]);
2969     result = ht.get(get);
2970     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2971         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2972 
2973     get = new Get(ROWS[ROWIDX]);
2974     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
2975     result = ht.get(get);
2976     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2977         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2978 
2979     get = new Get(ROWS[ROWIDX]);
2980     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
2981     get.addFamily(FAMILIES[FAMILYIDX]);
2982     result = ht.get(get);
2983     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2984         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2985 
2986     get = new Get(ROWS[ROWIDX]);
2987     get.addFamily(FAMILIES[FAMILYIDX]);
2988     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
2989     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
2990     get.addFamily(FAMILIES[FAMILYIDX-1]);
2991     get.addFamily(FAMILIES[FAMILYIDX+2]);
2992     result = ht.get(get);
2993     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
2994         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
2995 
2996   }
2997 
2998 
2999   /**
3000    * Verify a single column using scanners.
3001    * Expects family and qualifier arrays to be valid for at least
3002    * the range:  idx-2 to idx+2
3003    * Expects row array to be valid for at least idx to idx+2
3004    */
3005   private void scanVerifySingleColumn(Table ht,
3006       byte [][] ROWS, int ROWIDX,
3007       byte [][] FAMILIES, int FAMILYIDX,
3008       byte [][] QUALIFIERS, int QUALIFIERIDX,
3009       byte [][] VALUES, int VALUEIDX)
3010   throws Exception {
3011 
3012     Scan scan = new Scan();
3013     Result result = getSingleScanResult(ht, scan);
3014     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3015         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3016 
3017     scan = new Scan(ROWS[ROWIDX]);
3018     result = getSingleScanResult(ht, scan);
3019     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3020         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3021 
3022     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
3023     result = getSingleScanResult(ht, scan);
3024     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3025         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3026 
3027     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
3028     result = getSingleScanResult(ht, scan);
3029     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3030         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3031 
3032     scan = new Scan();
3033     scan.addFamily(FAMILIES[FAMILYIDX]);
3034     result = getSingleScanResult(ht, scan);
3035     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3036         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3037 
3038     scan = new Scan();
3039     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3040     result = getSingleScanResult(ht, scan);
3041     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3042         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3043 
3044     scan = new Scan();
3045     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3046     scan.addFamily(FAMILIES[FAMILYIDX]);
3047     result = getSingleScanResult(ht, scan);
3048     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3049         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3050 
3051     scan = new Scan();
3052     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
3053     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3054     scan.addFamily(FAMILIES[FAMILYIDX+1]);
3055     result = getSingleScanResult(ht, scan);
3056     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3057         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3058 
3059   }
3060 
3061   /**
3062    * Verify we do not read any values by accident around a single column
3063    * Same requirements as getVerifySingleColumn
3064    */
3065   private void getVerifySingleEmpty(Table ht,
3066       byte [][] ROWS, int ROWIDX,
3067       byte [][] FAMILIES, int FAMILYIDX,
3068       byte [][] QUALIFIERS, int QUALIFIERIDX)
3069   throws Exception {
3070 
3071     Get get = new Get(ROWS[ROWIDX]);
3072     get.addFamily(FAMILIES[4]);
3073     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
3074     Result result = ht.get(get);
3075     assertEmptyResult(result);
3076 
3077     get = new Get(ROWS[ROWIDX]);
3078     get.addFamily(FAMILIES[4]);
3079     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3080     result = ht.get(get);
3081     assertEmptyResult(result);
3082 
3083     get = new Get(ROWS[ROWIDX]);
3084     get.addFamily(FAMILIES[3]);
3085     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3086     get.addFamily(FAMILIES[5]);
3087     result = ht.get(get);
3088     assertEmptyResult(result);
3089 
3090     get = new Get(ROWS[ROWIDX+1]);
3091     result = ht.get(get);
3092     assertEmptyResult(result);
3093 
3094   }
3095 
3096   private void scanVerifySingleEmpty(Table ht,
3097       byte [][] ROWS, int ROWIDX,
3098       byte [][] FAMILIES, int FAMILYIDX,
3099       byte [][] QUALIFIERS, int QUALIFIERIDX)
3100   throws Exception {
3101 
3102     Scan scan = new Scan(ROWS[ROWIDX+1]);
3103     Result result = getSingleScanResult(ht, scan);
3104     assertNullResult(result);
3105 
3106     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
3107     result = getSingleScanResult(ht, scan);
3108     assertNullResult(result);
3109 
3110     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
3111     result = getSingleScanResult(ht, scan);
3112     assertNullResult(result);
3113 
3114     scan = new Scan();
3115     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3116     scan.addFamily(FAMILIES[FAMILYIDX-1]);
3117     result = getSingleScanResult(ht, scan);
3118     assertNullResult(result);
3119 
3120   }
3121 
3122   //
3123   // Verifiers
3124   //
3125 
3126   private void assertKey(Cell key, byte [] row, byte [] family,
3127       byte [] qualifier, byte [] value)
3128   throws Exception {
3129     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3130         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3131         equals(row, CellUtil.cloneRow(key)));
3132     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3133         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3134         equals(family, CellUtil.cloneFamily(key)));
3135     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3136         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3137         equals(qualifier, CellUtil.cloneQualifier(key)));
3138     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3139         "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3140         equals(value, CellUtil.cloneValue(key)));
3141   }
3142 
3143   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
3144       byte [] qualifier, long value)
3145   throws Exception {
3146     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3147         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3148         equals(row, CellUtil.cloneRow(key)));
3149     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3150         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3151         equals(family, CellUtil.cloneFamily(key)));
3152     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3153         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3154         equals(qualifier, CellUtil.cloneQualifier(key)));
3155     assertTrue("Expected value [" + value + "] " +
3156         "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
3157         Bytes.toLong(CellUtil.cloneValue(key)) == value);
3158   }
3159 
3160   private void assertNumKeys(Result result, int n) throws Exception {
3161     assertTrue("Expected " + n + " keys but got " + result.size(),
3162         result.size() == n);
3163   }
3164 
3165   private void assertNResult(Result result, byte [] row,
3166       byte [][] families, byte [][] qualifiers, byte [][] values,
3167       int [][] idxs)
3168   throws Exception {
3169     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3170         "Got row [" + Bytes.toString(result.getRow()) +"]",
3171         equals(row, result.getRow()));
3172     assertTrue("Expected " + idxs.length + " keys but result contains "
3173         + result.size(), result.size() == idxs.length);
3174 
3175     Cell [] keys = result.rawCells();
3176 
3177     for(int i=0;i<keys.length;i++) {
3178       byte [] family = families[idxs[i][0]];
3179       byte [] qualifier = qualifiers[idxs[i][1]];
3180       byte [] value = values[idxs[i][2]];
3181       Cell key = keys[i];
3182 
3183       byte[] famb = CellUtil.cloneFamily(key);
3184       byte[] qualb = CellUtil.cloneQualifier(key);
3185       byte[] valb = CellUtil.cloneValue(key);
3186       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3187           + "] " + "Got family [" + Bytes.toString(famb) + "]",
3188           equals(family, famb));
3189       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3190           + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
3191           equals(qualifier, qualb));
3192       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3193           + "Got value [" + Bytes.toString(valb) + "]",
3194           equals(value, valb));
3195     }
3196   }
3197 
3198   private void assertNResult(Result result, byte [] row,
3199       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3200       int start, int end)
3201   throws IOException {
3202     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3203         "Got row [" + Bytes.toString(result.getRow()) +"]",
3204         equals(row, result.getRow()));
3205     int expectedResults = end - start + 1;
3206     assertEquals(expectedResults, result.size());
3207 
3208     Cell[] keys = result.rawCells();
3209 
3210     for (int i=0; i<keys.length; i++) {
3211       byte [] value = values[end-i];
3212       long ts = stamps[end-i];
3213       Cell key = keys[i];
3214 
3215       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3216           + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3217           CellUtil.matchingFamily(key, family));
3218       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3219           + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
3220           CellUtil.matchingQualifier(key, qualifier));
3221       assertTrue("Expected ts [" + ts + "] " +
3222           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3223       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3224           + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3225           CellUtil.matchingValue(key,  value));
3226     }
3227   }
3228 
3229   /**
3230    * Validate that result contains two specified keys, exactly.
3231    * It is assumed key A sorts before key B.
3232    */
3233   private void assertDoubleResult(Result result, byte [] row,
3234       byte [] familyA, byte [] qualifierA, byte [] valueA,
3235       byte [] familyB, byte [] qualifierB, byte [] valueB)
3236   throws Exception {
3237     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3238         "Got row [" + Bytes.toString(result.getRow()) +"]",
3239         equals(row, result.getRow()));
3240     assertTrue("Expected two keys but result contains " + result.size(),
3241         result.size() == 2);
3242     Cell [] kv = result.rawCells();
3243     Cell kvA = kv[0];
3244     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3245         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
3246         equals(familyA, CellUtil.cloneFamily(kvA)));
3247     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3248         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
3249         equals(qualifierA, CellUtil.cloneQualifier(kvA)));
3250     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3251         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
3252         equals(valueA, CellUtil.cloneValue(kvA)));
3253     Cell kvB = kv[1];
3254     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3255         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
3256         equals(familyB, CellUtil.cloneFamily(kvB)));
3257     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3258         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
3259         equals(qualifierB, CellUtil.cloneQualifier(kvB)));
3260     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3261         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
3262         equals(valueB, CellUtil.cloneValue(kvB)));
3263   }
3264 
3265   private void assertSingleResult(Result result, byte [] row, byte [] family,
3266       byte [] qualifier, byte [] value)
3267   throws Exception {
3268     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3269         "Got row [" + Bytes.toString(result.getRow()) +"]",
3270         equals(row, result.getRow()));
3271     assertTrue("Expected a single key but result contains " + result.size(),
3272         result.size() == 1);
3273     Cell kv = result.rawCells()[0];
3274     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3275         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3276         equals(family, CellUtil.cloneFamily(kv)));
3277     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3278         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3279         equals(qualifier, CellUtil.cloneQualifier(kv)));
3280     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3281         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3282         equals(value, CellUtil.cloneValue(kv)));
3283   }
3284 
3285   private void assertSingleResult(Result result, byte [] row, byte [] family,
3286       byte [] qualifier, long ts, byte [] value)
3287   throws Exception {
3288     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3289         "Got row [" + Bytes.toString(result.getRow()) +"]",
3290         equals(row, result.getRow()));
3291     assertTrue("Expected a single key but result contains " + result.size(),
3292         result.size() == 1);
3293     Cell kv = result.rawCells()[0];
3294     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3295         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3296         equals(family, CellUtil.cloneFamily(kv)));
3297     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3298         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3299         equals(qualifier, CellUtil.cloneQualifier(kv)));
3300     assertTrue("Expected ts [" + ts + "] " +
3301         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3302     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3303         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3304         equals(value, CellUtil.cloneValue(kv)));
3305   }
3306 
3307   private void assertEmptyResult(Result result) throws Exception {
3308     assertTrue("expected an empty result but result contains " +
3309         result.size() + " keys", result.isEmpty());
3310   }
3311 
3312   private void assertNullResult(Result result) throws Exception {
3313     assertTrue("expected null result but received a non-null result",
3314         result == null);
3315   }
3316 
3317   //
3318   // Helpers
3319   //
3320 
3321   private Result getSingleScanResult(Table ht, Scan scan) throws IOException {
3322     ResultScanner scanner = ht.getScanner(scan);
3323     Result result = scanner.next();
3324     scanner.close();
3325     return result;
3326   }
3327 
3328   private byte [][] makeNAscii(byte [] base, int n) {
3329     if(n > 256) {
3330       return makeNBig(base, n);
3331     }
3332     byte [][] ret = new byte[n][];
3333     for(int i=0;i<n;i++) {
3334       byte [] tail = Bytes.toBytes(Integer.toString(i));
3335       ret[i] = Bytes.add(base, tail);
3336     }
3337     return ret;
3338   }
3339 
3340   private byte [][] makeN(byte [] base, int n) {
3341     if (n > 256) {
3342       return makeNBig(base, n);
3343     }
3344     byte [][] ret = new byte[n][];
3345     for(int i=0;i<n;i++) {
3346       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3347     }
3348     return ret;
3349   }
3350 
3351   private byte [][] makeNBig(byte [] base, int n) {
3352     byte [][] ret = new byte[n][];
3353     for(int i=0;i<n;i++) {
3354       int byteA = (i % 256);
3355       int byteB = (i >> 8);
3356       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3357     }
3358     return ret;
3359   }
3360 
3361   private long [] makeStamps(int n) {
3362     long [] stamps = new long[n];
3363     for(int i=0;i<n;i++) stamps[i] = i+1;
3364     return stamps;
3365   }
3366 
3367   static boolean equals(byte [] left, byte [] right) {
3368     if (left == null && right == null) return true;
3369     if (left == null && right.length == 0) return true;
3370     if (right == null && left.length == 0) return true;
3371     return Bytes.equals(left, right);
3372   }
3373 
3374   @Test
3375   public void testDuplicateVersions() throws Exception {
3376     byte [] TABLE = Bytes.toBytes("testDuplicateVersions");
3377 
3378     long [] STAMPS = makeStamps(20);
3379     byte [][] VALUES = makeNAscii(VALUE, 20);
3380 
3381     Table ht = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3382 
3383     // Insert 4 versions of same column
3384     Put put = new Put(ROW);
3385     put.add(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3386     put.add(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3387     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3388     put.add(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3389     ht.put(put);
3390 
3391     // Verify we can get each one properly
3392     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3393     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3394     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3395     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3396     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3397     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3398     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3399     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3400 
3401     // Verify we don't accidentally get others
3402     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3403     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3404     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3405     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3406     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3407     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3408 
3409     // Ensure maxVersions in query is respected
3410     Get get = new Get(ROW);
3411     get.addColumn(FAMILY, QUALIFIER);
3412     get.setMaxVersions(2);
3413     Result result = ht.get(get);
3414     assertNResult(result, ROW, FAMILY, QUALIFIER,
3415         new long [] {STAMPS[4], STAMPS[5]},
3416         new byte[][] {VALUES[4], VALUES[5]},
3417         0, 1);
3418 
3419     Scan scan = new Scan(ROW);
3420     scan.addColumn(FAMILY, QUALIFIER);
3421     scan.setMaxVersions(2);
3422     result = getSingleScanResult(ht, scan);
3423     assertNResult(result, ROW, FAMILY, QUALIFIER,
3424         new long [] {STAMPS[4], STAMPS[5]},
3425         new byte[][] {VALUES[4], VALUES[5]},
3426         0, 1);
3427 
3428     // Flush and redo
3429 
3430     TEST_UTIL.flush();
3431 
3432     // Verify we can get each one properly
3433     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3434     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3435     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3436     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3437     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3438     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3439     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3440     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3441 
3442     // Verify we don't accidentally get others
3443     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3444     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3445     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3446     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3447     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3448     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3449 
3450     // Ensure maxVersions in query is respected
3451     get = new Get(ROW);
3452     get.addColumn(FAMILY, QUALIFIER);
3453     get.setMaxVersions(2);
3454     result = ht.get(get);
3455     assertNResult(result, ROW, FAMILY, QUALIFIER,
3456         new long [] {STAMPS[4], STAMPS[5]},
3457         new byte[][] {VALUES[4], VALUES[5]},
3458         0, 1);
3459 
3460     scan = new Scan(ROW);
3461     scan.addColumn(FAMILY, QUALIFIER);
3462     scan.setMaxVersions(2);
3463     result = getSingleScanResult(ht, scan);
3464     assertNResult(result, ROW, FAMILY, QUALIFIER,
3465         new long [] {STAMPS[4], STAMPS[5]},
3466         new byte[][] {VALUES[4], VALUES[5]},
3467         0, 1);
3468 
3469 
3470     // Add some memstore and retest
3471 
3472     // Insert 4 more versions of same column and a dupe
3473     put = new Put(ROW);
3474     put.add(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3475     put.add(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3476     put.add(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3477     put.add(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3478     put.add(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3479     ht.put(put);
3480 
3481     // Ensure maxVersions in query is respected
3482     get = new Get(ROW);
3483     get.addColumn(FAMILY, QUALIFIER);
3484     get.setMaxVersions(7);
3485     result = ht.get(get);
3486     assertNResult(result, ROW, FAMILY, QUALIFIER,
3487         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3488         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3489         0, 6);
3490 
3491     scan = new Scan(ROW);
3492     scan.addColumn(FAMILY, QUALIFIER);
3493     scan.setMaxVersions(7);
3494     result = getSingleScanResult(ht, scan);
3495     assertNResult(result, ROW, FAMILY, QUALIFIER,
3496         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3497         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3498         0, 6);
3499 
3500     get = new Get(ROW);
3501     get.setMaxVersions(7);
3502     result = ht.get(get);
3503     assertNResult(result, ROW, FAMILY, QUALIFIER,
3504         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3505         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3506         0, 6);
3507 
3508     scan = new Scan(ROW);
3509     scan.setMaxVersions(7);
3510     result = getSingleScanResult(ht, scan);
3511     assertNResult(result, ROW, FAMILY, QUALIFIER,
3512         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3513         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3514         0, 6);
3515 
3516     // Verify we can get each one properly
3517     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3518     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3519     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3520     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3521     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3522     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3523     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3524     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3525 
3526     // Verify we don't accidentally get others
3527     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3528     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3529     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3530     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3531 
3532     // Ensure maxVersions of table is respected
3533 
3534     TEST_UTIL.flush();
3535 
3536     // Insert 4 more versions of same column and a dupe
3537     put = new Put(ROW);
3538     put.add(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3539     put.add(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3540     put.add(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3541     put.add(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3542     ht.put(put);
3543 
3544     get = new Get(ROW);
3545     get.addColumn(FAMILY, QUALIFIER);
3546     get.setMaxVersions(Integer.MAX_VALUE);
3547     result = ht.get(get);
3548     assertNResult(result, ROW, FAMILY, QUALIFIER,
3549         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3550         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3551         0, 9);
3552 
3553     scan = new Scan(ROW);
3554     scan.addColumn(FAMILY, QUALIFIER);
3555     scan.setMaxVersions(Integer.MAX_VALUE);
3556     result = getSingleScanResult(ht, scan);
3557     assertNResult(result, ROW, FAMILY, QUALIFIER,
3558         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3559         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3560         0, 9);
3561 
3562     // Delete a version in the memstore and a version in a storefile
3563     Delete delete = new Delete(ROW);
3564     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[11]);
3565     delete.deleteColumn(FAMILY, QUALIFIER, STAMPS[7]);
3566     ht.delete(delete);
3567 
3568     // Test that it's gone
3569     get = new Get(ROW);
3570     get.addColumn(FAMILY, QUALIFIER);
3571     get.setMaxVersions(Integer.MAX_VALUE);
3572     result = ht.get(get);
3573     assertNResult(result, ROW, FAMILY, QUALIFIER,
3574         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3575         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3576         0, 9);
3577 
3578     scan = new Scan(ROW);
3579     scan.addColumn(FAMILY, QUALIFIER);
3580     scan.setMaxVersions(Integer.MAX_VALUE);
3581     result = getSingleScanResult(ht, scan);
3582     assertNResult(result, ROW, FAMILY, QUALIFIER,
3583         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3584         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3585         0, 9);
3586   }
3587 
3588   @Test
3589   public void testUpdates() throws Exception {
3590 
3591     byte [] TABLE = Bytes.toBytes("testUpdates");
3592     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3593 
3594     // Write a column with values at timestamp 1, 2 and 3
3595     byte[] row = Bytes.toBytes("row1");
3596     byte[] qualifier = Bytes.toBytes("myCol");
3597     Put put = new Put(row);
3598     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3599     hTable.put(put);
3600 
3601     put = new Put(row);
3602     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3603     hTable.put(put);
3604 
3605     put = new Put(row);
3606     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3607     hTable.put(put);
3608 
3609     Get get = new Get(row);
3610     get.addColumn(FAMILY, qualifier);
3611     get.setMaxVersions();
3612 
3613     // Check that the column indeed has the right values at timestamps 1 and
3614     // 2
3615     Result result = hTable.get(get);
3616     NavigableMap<Long, byte[]> navigableMap =
3617         result.getMap().get(FAMILY).get(qualifier);
3618     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3619     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3620 
3621     // Update the value at timestamp 1
3622     put = new Put(row);
3623     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3624     hTable.put(put);
3625 
3626     // Update the value at timestamp 2
3627     put = new Put(row);
3628     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3629     hTable.put(put);
3630 
3631     // Check that the values at timestamp 2 and 1 got updated
3632     result = hTable.get(get);
3633     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3634     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3635     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3636   }
3637 
3638   @Test
3639   public void testUpdatesWithMajorCompaction() throws Exception {
3640 
3641     TableName TABLE = TableName.valueOf("testUpdatesWithMajorCompaction");
3642     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3643     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3644 
3645     // Write a column with values at timestamp 1, 2 and 3
3646     byte[] row = Bytes.toBytes("row2");
3647     byte[] qualifier = Bytes.toBytes("myCol");
3648     Put put = new Put(row);
3649     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3650     hTable.put(put);
3651 
3652     put = new Put(row);
3653     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3654     hTable.put(put);
3655 
3656     put = new Put(row);
3657     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3658     hTable.put(put);
3659 
3660     Get get = new Get(row);
3661     get.addColumn(FAMILY, qualifier);
3662     get.setMaxVersions();
3663 
3664     // Check that the column indeed has the right values at timestamps 1 and
3665     // 2
3666     Result result = hTable.get(get);
3667     NavigableMap<Long, byte[]> navigableMap =
3668         result.getMap().get(FAMILY).get(qualifier);
3669     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3670     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3671 
3672     // Trigger a major compaction
3673     admin.flush(TABLE);
3674     admin.majorCompact(TABLE);
3675     Thread.sleep(6000);
3676 
3677     // Update the value at timestamp 1
3678     put = new Put(row);
3679     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3680     hTable.put(put);
3681 
3682     // Update the value at timestamp 2
3683     put = new Put(row);
3684     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3685     hTable.put(put);
3686 
3687     // Trigger a major compaction
3688     admin.flush(TABLE);
3689     admin.majorCompact(TABLE);
3690     Thread.sleep(6000);
3691 
3692     // Check that the values at timestamp 2 and 1 got updated
3693     result = hTable.get(get);
3694     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3695     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3696     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3697   }
3698 
3699   @Test
3700   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3701 
3702     String tableName = "testMajorCompactionBetweenTwoUpdates";
3703     byte [] TABLE = Bytes.toBytes(tableName);
3704     Table hTable = TEST_UTIL.createTable(TABLE, FAMILY, 10);
3705     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
3706 
3707     // Write a column with values at timestamp 1, 2 and 3
3708     byte[] row = Bytes.toBytes("row3");
3709     byte[] qualifier = Bytes.toBytes("myCol");
3710     Put put = new Put(row);
3711     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3712     hTable.put(put);
3713 
3714     put = new Put(row);
3715     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3716     hTable.put(put);
3717 
3718     put = new Put(row);
3719     put.add(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3720     hTable.put(put);
3721 
3722     Get get = new Get(row);
3723     get.addColumn(FAMILY, qualifier);
3724     get.setMaxVersions();
3725 
3726     // Check that the column indeed has the right values at timestamps 1 and
3727     // 2
3728     Result result = hTable.get(get);
3729     NavigableMap<Long, byte[]> navigableMap =
3730         result.getMap().get(FAMILY).get(qualifier);
3731     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3732     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3733 
3734     // Trigger a major compaction
3735     admin.flush(tableName);
3736     admin.majorCompact(tableName);
3737     Thread.sleep(6000);
3738 
3739     // Update the value at timestamp 1
3740     put = new Put(row);
3741     put.add(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3742     hTable.put(put);
3743 
3744     // Trigger a major compaction
3745     admin.flush(tableName);
3746     admin.majorCompact(tableName);
3747     Thread.sleep(6000);
3748 
3749     // Update the value at timestamp 2
3750     put = new Put(row);
3751     put.add(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3752     hTable.put(put);
3753 
3754     // Trigger a major compaction
3755     admin.flush(tableName);
3756     admin.majorCompact(tableName);
3757     Thread.sleep(6000);
3758 
3759     // Check that the values at timestamp 2 and 1 got updated
3760     result = hTable.get(get);
3761     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3762 
3763     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3764     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3765   }
3766 
3767   @Test
3768   public void testGet_EmptyTable() throws IOException {
3769     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_EmptyTable"), FAMILY);
3770     Get get = new Get(ROW);
3771     get.addFamily(FAMILY);
3772     Result r = table.get(get);
3773     assertTrue(r.isEmpty());
3774   }
3775 
3776   @Test
3777   public void testGet_NullQualifier() throws IOException {
3778     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_NullQualifier"), FAMILY);
3779     Put put = new Put(ROW);
3780     put.add(FAMILY, QUALIFIER, VALUE);
3781     table.put(put);
3782 
3783     put = new Put(ROW);
3784     put.add(FAMILY, null, VALUE);
3785     table.put(put);
3786     LOG.info("Row put");
3787 
3788     Get get = new Get(ROW);
3789     get.addColumn(FAMILY, null);
3790     Result r = table.get(get);
3791     assertEquals(1, r.size());
3792 
3793     get = new Get(ROW);
3794     get.addFamily(FAMILY);
3795     r = table.get(get);
3796     assertEquals(2, r.size());
3797   }
3798 
3799   @Test
3800   public void testGet_NonExistentRow() throws IOException {
3801     Table table = TEST_UTIL.createTable(TableName.valueOf("testGet_NonExistentRow"), FAMILY);
3802     Put put = new Put(ROW);
3803     put.add(FAMILY, QUALIFIER, VALUE);
3804     table.put(put);
3805     LOG.info("Row put");
3806 
3807     Get get = new Get(ROW);
3808     get.addFamily(FAMILY);
3809     Result r = table.get(get);
3810     assertFalse(r.isEmpty());
3811     System.out.println("Row retrieved successfully");
3812 
3813     byte [] missingrow = Bytes.toBytes("missingrow");
3814     get = new Get(missingrow);
3815     get.addFamily(FAMILY);
3816     r = table.get(get);
3817     assertTrue(r.isEmpty());
3818     LOG.info("Row missing as it should be");
3819   }
3820 
3821   @Test
3822   public void testPut() throws IOException {
3823     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3824     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3825     final byte [] row1 = Bytes.toBytes("row1");
3826     final byte [] row2 = Bytes.toBytes("row2");
3827     final byte [] value = Bytes.toBytes("abcd");
3828     Table table = TEST_UTIL.createTable(Bytes.toBytes("testPut"),
3829       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3830     Put put = new Put(row1);
3831     put.add(CONTENTS_FAMILY, null, value);
3832     table.put(put);
3833 
3834     put = new Put(row2);
3835     put.add(CONTENTS_FAMILY, null, value);
3836 
3837     assertEquals(put.size(), 1);
3838     assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);
3839 
3840     // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
3841     KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
3842 
3843     assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY));
3844     // will it return null or an empty byte array?
3845     assertTrue(Bytes.equals(kv.getQualifier(), new byte[0]));
3846 
3847     assertTrue(Bytes.equals(kv.getValue(), value));
3848 
3849     table.put(put);
3850 
3851     Scan scan = new Scan();
3852     scan.addColumn(CONTENTS_FAMILY, null);
3853     ResultScanner scanner = table.getScanner(scan);
3854     for (Result r : scanner) {
3855       for(Cell key : r.rawCells()) {
3856         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
3857       }
3858     }
3859   }
3860 
3861   @Test
3862   public void testPutNoCF() throws IOException {
3863     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
3864     final byte[] VAL = Bytes.toBytes(100);
3865     Table table = TEST_UTIL.createTable(Bytes.toBytes("testPutNoCF"), FAMILY);
3866 
3867     boolean caughtNSCFE = false;
3868 
3869     try {
3870       Put p = new Put(ROW);
3871       p.add(BAD_FAM, QUALIFIER, VAL);
3872       table.put(p);
3873     } catch (RetriesExhaustedWithDetailsException e) {
3874       caughtNSCFE = e.getCause(0) instanceof NoSuchColumnFamilyException;
3875     }
3876     assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
3877 
3878   }
3879 
3880   @Test
3881   public void testRowsPut() throws IOException {
3882     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3883     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3884     final int NB_BATCH_ROWS = 10;
3885     final byte[] value = Bytes.toBytes("abcd");
3886     Table table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPut"),
3887       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3888     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3889     for (int i = 0; i < NB_BATCH_ROWS; i++) {
3890       byte[] row = Bytes.toBytes("row" + i);
3891       Put put = new Put(row);
3892       put.setDurability(Durability.SKIP_WAL);
3893       put.add(CONTENTS_FAMILY, null, value);
3894       rowsUpdate.add(put);
3895     }
3896     table.put(rowsUpdate);
3897     Scan scan = new Scan();
3898     scan.addFamily(CONTENTS_FAMILY);
3899     ResultScanner scanner = table.getScanner(scan);
3900     int nbRows = 0;
3901     for (@SuppressWarnings("unused")
3902     Result row : scanner)
3903       nbRows++;
3904     assertEquals(NB_BATCH_ROWS, nbRows);
3905   }
3906 
3907   @Test
3908   public void testRowsPutBufferedOneFlush() throws IOException {
3909     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3910     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3911     final byte [] value = Bytes.toBytes("abcd");
3912     final int NB_BATCH_ROWS = 10;
3913     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedOneFlush"),
3914       new byte [][] {CONTENTS_FAMILY, SMALL_FAMILY});
3915     table.setAutoFlush(false);
3916     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3917     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3918       byte[] row = Bytes.toBytes("row" + i);
3919       Put put = new Put(row);
3920       put.setDurability(Durability.SKIP_WAL);
3921       put.add(CONTENTS_FAMILY, null, value);
3922       rowsUpdate.add(put);
3923     }
3924     table.put(rowsUpdate);
3925 
3926     Scan scan = new Scan();
3927     scan.addFamily(CONTENTS_FAMILY);
3928     ResultScanner scanner = table.getScanner(scan);
3929     int nbRows = 0;
3930     for (@SuppressWarnings("unused")
3931     Result row : scanner)
3932       nbRows++;
3933     assertEquals(0, nbRows);
3934     scanner.close();
3935 
3936     table.flushCommits();
3937 
3938     scan = new Scan();
3939     scan.addFamily(CONTENTS_FAMILY);
3940     scanner = table.getScanner(scan);
3941     nbRows = 0;
3942     for (@SuppressWarnings("unused")
3943     Result row : scanner)
3944       nbRows++;
3945     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3946     table.close();
3947   }
3948 
3949   @Test
3950   public void testRowsPutBufferedManyManyFlushes() throws IOException {
3951     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3952     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
3953     final byte[] value = Bytes.toBytes("abcd");
3954     final int NB_BATCH_ROWS = 10;
3955     HTable table = TEST_UTIL.createTable(Bytes.toBytes("testRowsPutBufferedManyManyFlushes"),
3956       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
3957     table.setWriteBufferSize(10);
3958     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
3959     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
3960       byte[] row = Bytes.toBytes("row" + i);
3961       Put put = new Put(row);
3962       put.setDurability(Durability.SKIP_WAL);
3963       put.add(CONTENTS_FAMILY, null, value);
3964       rowsUpdate.add(put);
3965     }
3966     table.put(rowsUpdate);
3967 
3968     Scan scan = new Scan();
3969     scan.addFamily(CONTENTS_FAMILY);
3970     ResultScanner scanner = table.getScanner(scan);
3971     int nbRows = 0;
3972     for (@SuppressWarnings("unused")
3973     Result row : scanner)
3974       nbRows++;
3975     assertEquals(NB_BATCH_ROWS * 10, nbRows);
3976   }
3977 
3978   @Test
3979   public void testAddKeyValue() throws IOException {
3980     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
3981     final byte[] value = Bytes.toBytes("abcd");
3982     final byte[] row1 = Bytes.toBytes("row1");
3983     final byte[] row2 = Bytes.toBytes("row2");
3984     byte[] qualifier = Bytes.toBytes("qf1");
3985     Put put = new Put(row1);
3986 
3987     // Adding KeyValue with the same row
3988     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
3989     boolean ok = true;
3990     try {
3991       put.add(kv);
3992     } catch (IOException e) {
3993       ok = false;
3994     }
3995     assertEquals(true, ok);
3996 
3997     // Adding KeyValue with the different row
3998     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
3999     ok = false;
4000     try {
4001       put.add(kv);
4002     } catch (IOException e) {
4003       ok = true;
4004     }
4005     assertEquals(true, ok);
4006   }
4007 
4008   /**
4009    * test for HBASE-737
4010    * @throws IOException
4011    */
4012   @Test
4013   public void testHBase737 () throws IOException {
4014     final byte [] FAM1 = Bytes.toBytes("fam1");
4015     final byte [] FAM2 = Bytes.toBytes("fam2");
4016     // Open table
4017     Table table = TEST_UTIL.createTable(Bytes.toBytes("testHBase737"),
4018       new byte [][] {FAM1, FAM2});
4019     // Insert some values
4020     Put put = new Put(ROW);
4021     put.add(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
4022     table.put(put);
4023     try {
4024       Thread.sleep(1000);
4025     } catch (InterruptedException i) {
4026       //ignore
4027     }
4028 
4029     put = new Put(ROW);
4030     put.add(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
4031     table.put(put);
4032 
4033     try {
4034       Thread.sleep(1000);
4035     } catch (InterruptedException i) {
4036       //ignore
4037     }
4038 
4039     put = new Put(ROW);
4040     put.add(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
4041     table.put(put);
4042 
4043     long times[] = new long[3];
4044 
4045     // First scan the memstore
4046 
4047     Scan scan = new Scan();
4048     scan.addFamily(FAM1);
4049     scan.addFamily(FAM2);
4050     ResultScanner s = table.getScanner(scan);
4051     try {
4052       int index = 0;
4053       Result r = null;
4054       while ((r = s.next()) != null) {
4055         for(Cell key : r.rawCells()) {
4056           times[index++] = key.getTimestamp();
4057         }
4058       }
4059     } finally {
4060       s.close();
4061     }
4062     for (int i = 0; i < times.length - 1; i++) {
4063       for (int j = i + 1; j < times.length; j++) {
4064         assertTrue(times[j] > times[i]);
4065       }
4066     }
4067 
4068     // Flush data to disk and try again
4069     TEST_UTIL.flush();
4070 
4071     // Reset times
4072     for(int i=0;i<times.length;i++) {
4073       times[i] = 0;
4074     }
4075 
4076     try {
4077       Thread.sleep(1000);
4078     } catch (InterruptedException i) {
4079       //ignore
4080     }
4081     scan = new Scan();
4082     scan.addFamily(FAM1);
4083     scan.addFamily(FAM2);
4084     s = table.getScanner(scan);
4085     try {
4086       int index = 0;
4087       Result r = null;
4088       while ((r = s.next()) != null) {
4089         for(Cell key : r.rawCells()) {
4090           times[index++] = key.getTimestamp();
4091         }
4092       }
4093     } finally {
4094       s.close();
4095     }
4096     for (int i = 0; i < times.length - 1; i++) {
4097       for (int j = i + 1; j < times.length; j++) {
4098         assertTrue(times[j] > times[i]);
4099       }
4100     }
4101   }
4102 
4103   @Test
4104   public void testListTables() throws IOException, InterruptedException {
4105     TableName t1 = TableName.valueOf("testListTables1");
4106     TableName t2 = TableName.valueOf("testListTables2");
4107     TableName t3 = TableName.valueOf("testListTables3");
4108     TableName [] tables = new TableName[] { t1, t2, t3 };
4109     for (int i = 0; i < tables.length; i++) {
4110       TEST_UTIL.createTable(tables[i], FAMILY);
4111     }
4112     Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4113     HTableDescriptor[] ts = admin.listTables();
4114     HashSet<HTableDescriptor> result = new HashSet<HTableDescriptor>(ts.length);
4115     Collections.addAll(result, ts);
4116     int size = result.size();
4117     assertTrue(size >= tables.length);
4118     for (int i = 0; i < tables.length && i < size; i++) {
4119       boolean found = false;
4120       for (int j = 0; j < ts.length; j++) {
4121         if (ts[j].getTableName().equals(tables[i])) {
4122           found = true;
4123           break;
4124         }
4125       }
4126       assertTrue("Not found: " + tables[i], found);
4127     }
4128   }
4129 
4130   /**
4131    * creates an HTable for tableName using an unmanaged HConnection.
4132    *
4133    * @param tableName - table to create
4134    * @return the created HTable object
4135    * @throws IOException
4136    */
4137   HTable createUnmangedHConnectionHTable(final TableName tableName) throws IOException {
4138     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4139     HConnection conn = HConnectionManager.createConnection(TEST_UTIL.getConfiguration());
4140     return (HTable)conn.getTable(tableName);
4141   }
4142 
4143   /**
4144    * simple test that just executes parts of the client
4145    * API that accept a pre-created HConnection instance
4146    *
4147    * @throws IOException
4148    */
4149   @Test
4150   public void testUnmanagedHConnection() throws IOException {
4151     final TableName tableName = TableName.valueOf("testUnmanagedHConnection");
4152     HTable t = createUnmangedHConnectionHTable(tableName);
4153     HBaseAdmin ha = new HBaseAdmin(t.getConnection());
4154     assertTrue(ha.tableExists(tableName));
4155     assertTrue(t.get(new Get(ROW)).isEmpty());
4156     ha.close();
4157   }
4158 
4159   /**
4160    * test of that unmanaged HConnections are able to reconnect
4161    * properly (see HBASE-5058)
4162    *
4163    * @throws Exception
4164    */
4165   @Test
4166   public void testUnmanagedHConnectionReconnect() throws Exception {
4167     final TableName tableName = TableName.valueOf("testUnmanagedHConnectionReconnect");
4168     HTable t = createUnmangedHConnectionHTable(tableName);
4169     Connection conn = t.getConnection();
4170     try (HBaseAdmin ha = new HBaseAdmin(conn)) {
4171       assertTrue(ha.tableExists(tableName));
4172       assertTrue(t.get(new Get(ROW)).isEmpty());
4173     }
4174 
4175     // stop the master
4176     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4177     cluster.stopMaster(0, false);
4178     cluster.waitOnMaster(0);
4179 
4180     // start up a new master
4181     cluster.startMaster();
4182     assertTrue(cluster.waitForActiveAndReadyMaster());
4183 
4184     // test that the same unmanaged connection works with a new
4185     // HBaseAdmin and can connect to the new master;
4186     try (HBaseAdmin newAdmin = new HBaseAdmin(conn)) {
4187       assertTrue(newAdmin.tableExists(tableName));
4188       assertTrue(newAdmin.getClusterStatus().getServersSize() == SLAVES);
4189     }
4190   }
4191 
4192   @Test
4193   public void testMiscHTableStuff() throws IOException {
4194     final TableName tableAname = TableName.valueOf("testMiscHTableStuffA");
4195     final TableName tableBname = TableName.valueOf("testMiscHTableStuffB");
4196     final byte[] attrName = Bytes.toBytes("TESTATTR");
4197     final byte[] attrValue = Bytes.toBytes("somevalue");
4198     byte[] value = Bytes.toBytes("value");
4199 
4200     Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4201     Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
4202     Put put = new Put(ROW);
4203     put.add(HConstants.CATALOG_FAMILY, null, value);
4204     a.put(put);
4205 
4206     // open a new connection to A and a connection to b
4207     Table newA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4208 
4209     // copy data from A to B
4210     Scan scan = new Scan();
4211     scan.addFamily(HConstants.CATALOG_FAMILY);
4212     ResultScanner s = newA.getScanner(scan);
4213     try {
4214       for (Result r : s) {
4215         put = new Put(r.getRow());
4216         put.setDurability(Durability.SKIP_WAL);
4217         for (Cell kv : r.rawCells()) {
4218           put.add(kv);
4219         }
4220         b.put(put);
4221       }
4222     } finally {
4223       s.close();
4224     }
4225 
4226     // Opening a new connection to A will cause the tables to be reloaded
4227     Table anotherA = new HTable(TEST_UTIL.getConfiguration(), tableAname);
4228     Get get = new Get(ROW);
4229     get.addFamily(HConstants.CATALOG_FAMILY);
4230     anotherA.get(get);
4231 
4232     // We can still access A through newA because it has the table information
4233     // cached. And if it needs to recalibrate, that will cause the information
4234     // to be reloaded.
4235 
4236     // Test user metadata
4237     Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
4238     // make a modifiable descriptor
4239     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
4240     // offline the table
4241     admin.disableTable(tableAname);
4242     // add a user attribute to HTD
4243     desc.setValue(attrName, attrValue);
4244     // add a user attribute to HCD
4245     for (HColumnDescriptor c : desc.getFamilies())
4246       c.setValue(attrName, attrValue);
4247     // update metadata for all regions of this table
4248     admin.modifyTable(tableAname, desc);
4249     // enable the table
4250     admin.enableTable(tableAname);
4251 
4252     // Test that attribute changes were applied
4253     desc = a.getTableDescriptor();
4254     assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
4255     // check HTD attribute
4256     value = desc.getValue(attrName);
4257     assertFalse("missing HTD attribute value", value == null);
4258     assertFalse("HTD attribute value is incorrect",
4259       Bytes.compareTo(value, attrValue) != 0);
4260     // check HCD attribute
4261     for (HColumnDescriptor c : desc.getFamilies()) {
4262       value = c.getValue(attrName);
4263       assertFalse("missing HCD attribute value", value == null);
4264       assertFalse("HCD attribute value is incorrect",
4265         Bytes.compareTo(value, attrValue) != 0);
4266     }
4267   }
4268 
4269   @Test
4270   public void testGetClosestRowBefore() throws IOException, InterruptedException {
4271     final TableName tableAname = TableName.valueOf("testGetClosestRowBefore");
4272     final byte[] firstRow = Bytes.toBytes("row111");
4273     final byte[] secondRow = Bytes.toBytes("row222");
4274     final byte[] thirdRow = Bytes.toBytes("row333");
4275     final byte[] forthRow = Bytes.toBytes("row444");
4276     final byte[] beforeFirstRow = Bytes.toBytes("row");
4277     final byte[] beforeSecondRow = Bytes.toBytes("row22");
4278     final byte[] beforeThirdRow = Bytes.toBytes("row33");
4279     final byte[] beforeForthRow = Bytes.toBytes("row44");
4280 
4281     HTable table =
4282         TEST_UTIL.createTable(tableAname,
4283           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
4284     // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
4285     // in Store.rowAtOrBeforeFromStoreFile
4286     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4287     Region region =
4288         TEST_UTIL.getRSForFirstRegionInTable(tableAname).getFromOnlineRegions(regionName);
4289     Put put1 = new Put(firstRow);
4290     Put put2 = new Put(secondRow);
4291     Put put3 = new Put(thirdRow);
4292     Put put4 = new Put(forthRow);
4293     byte[] one = new byte[] { 1 };
4294     byte[] two = new byte[] { 2 };
4295     byte[] three = new byte[] { 3 };
4296     byte[] four = new byte[] { 4 };
4297 
4298     put1.add(HConstants.CATALOG_FAMILY, null, one);
4299     put2.add(HConstants.CATALOG_FAMILY, null, two);
4300     put3.add(HConstants.CATALOG_FAMILY, null, three);
4301     put4.add(HConstants.CATALOG_FAMILY, null, four);
4302     table.put(put1);
4303     table.put(put2);
4304     table.put(put3);
4305     table.put(put4);
4306     region.flush(true);
4307     Result result = null;
4308 
4309     // Test before first that null is returned
4310     result = table.getRowOrBefore(beforeFirstRow, HConstants.CATALOG_FAMILY);
4311     assertTrue(result == null);
4312 
4313     // Test at first that first is returned
4314     result = table.getRowOrBefore(firstRow, HConstants.CATALOG_FAMILY);
4315     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4316     assertTrue(Bytes.equals(result.getRow(), firstRow));
4317     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4318 
4319     // Test in between first and second that first is returned
4320     result = table.getRowOrBefore(beforeSecondRow, HConstants.CATALOG_FAMILY);
4321     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4322     assertTrue(Bytes.equals(result.getRow(), firstRow));
4323     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4324 
4325     // Test at second make sure second is returned
4326     result = table.getRowOrBefore(secondRow, HConstants.CATALOG_FAMILY);
4327     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4328     assertTrue(Bytes.equals(result.getRow(), secondRow));
4329     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4330 
4331     // Test in second and third, make sure second is returned
4332     result = table.getRowOrBefore(beforeThirdRow, HConstants.CATALOG_FAMILY);
4333     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4334     assertTrue(Bytes.equals(result.getRow(), secondRow));
4335     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4336 
4337     // Test at third make sure third is returned
4338     result = table.getRowOrBefore(thirdRow, HConstants.CATALOG_FAMILY);
4339     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4340     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4341     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4342 
4343     // Test in third and forth, make sure third is returned
4344     result = table.getRowOrBefore(beforeForthRow, HConstants.CATALOG_FAMILY);
4345     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4346     assertTrue(Bytes.equals(result.getRow(), thirdRow));
4347     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4348 
4349     // Test at forth make sure forth is returned
4350     result = table.getRowOrBefore(forthRow, HConstants.CATALOG_FAMILY);
4351     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4352     assertTrue(Bytes.equals(result.getRow(), forthRow));
4353     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4354 
4355     // Test after forth make sure forth is returned
4356     result = table.getRowOrBefore(Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
4357     assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4358     assertTrue(Bytes.equals(result.getRow(), forthRow));
4359     assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4360 
4361     table.close();
4362   }
4363 
4364   /**
4365    * For HBASE-2156
4366    * @throws Exception
4367    */
4368   @Test
4369   public void testScanVariableReuse() throws Exception {
4370     Scan scan = new Scan();
4371     scan.addFamily(FAMILY);
4372     scan.addColumn(FAMILY, ROW);
4373 
4374     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4375 
4376     scan = new Scan();
4377     scan.addFamily(FAMILY);
4378 
4379     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4380     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4381   }
4382 
4383   @Test
4384   public void testMultiRowMutation() throws Exception {
4385     LOG.info("Starting testMultiRowMutation");
4386     final TableName TABLENAME = TableName.valueOf("testMultiRowMutation");
4387     final byte [] ROW1 = Bytes.toBytes("testRow1");
4388 
4389     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4390     Put p = new Put(ROW);
4391     p.add(FAMILY, QUALIFIER, VALUE);
4392     MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
4393 
4394     p = new Put(ROW1);
4395     p.add(FAMILY, QUALIFIER, VALUE);
4396     MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
4397 
4398     MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
4399     mrmBuilder.addMutationRequest(m1);
4400     mrmBuilder.addMutationRequest(m2);
4401     MutateRowsRequest mrm = mrmBuilder.build();
4402     CoprocessorRpcChannel channel = t.coprocessorService(ROW);
4403     MultiRowMutationService.BlockingInterface service =
4404        MultiRowMutationService.newBlockingStub(channel);
4405     service.mutateRows(null, mrm);
4406     Get g = new Get(ROW);
4407     Result r = t.get(g);
4408     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4409     g = new Get(ROW1);
4410     r = t.get(g);
4411     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4412   }
4413 
4414   @Test
4415   public void testRowMutation() throws Exception {
4416     LOG.info("Starting testRowMutation");
4417     final TableName TABLENAME = TableName.valueOf("testRowMutation");
4418     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4419     byte [][] QUALIFIERS = new byte [][] {
4420         Bytes.toBytes("a"), Bytes.toBytes("b")
4421     };
4422     RowMutations arm = new RowMutations(ROW);
4423     Put p = new Put(ROW);
4424     p.add(FAMILY, QUALIFIERS[0], VALUE);
4425     arm.add(p);
4426     t.mutateRow(arm);
4427 
4428     Get g = new Get(ROW);
4429     Result r = t.get(g);
4430     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4431 
4432     arm = new RowMutations(ROW);
4433     p = new Put(ROW);
4434     p.add(FAMILY, QUALIFIERS[1], VALUE);
4435     arm.add(p);
4436     Delete d = new Delete(ROW);
4437     d.deleteColumns(FAMILY, QUALIFIERS[0]);
4438     arm.add(d);
4439     // TODO: Trying mutateRow again.  The batch was failing with a one try only.
4440     t.mutateRow(arm);
4441     r = t.get(g);
4442     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4443     assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4444 
4445     //Test that we get a region level exception
4446     try {
4447       arm = new RowMutations(ROW);
4448       p = new Put(ROW);
4449       p.add(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
4450       arm.add(p);
4451       t.mutateRow(arm);
4452       fail("Expected NoSuchColumnFamilyException");
4453     } catch(NoSuchColumnFamilyException e) {
4454     }
4455   }
4456 
4457   @Test
4458   public void testAppend() throws Exception {
4459     LOG.info("Starting testAppend");
4460     final TableName TABLENAME = TableName.valueOf("testAppend");
4461     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4462     byte[] v1 = Bytes.toBytes("42");
4463     byte[] v2 = Bytes.toBytes("23");
4464     byte [][] QUALIFIERS = new byte [][] {
4465         Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
4466     };
4467     Append a = new Append(ROW);
4468     a.add(FAMILY, QUALIFIERS[0], v1);
4469     a.add(FAMILY, QUALIFIERS[1], v2);
4470     a.setReturnResults(false);
4471     assertNullResult(t.append(a));
4472 
4473     a = new Append(ROW);
4474     a.add(FAMILY, QUALIFIERS[0], v2);
4475     a.add(FAMILY, QUALIFIERS[1], v1);
4476     a.add(FAMILY, QUALIFIERS[2], v2);
4477     Result r = t.append(a);
4478     assertEquals(0, Bytes.compareTo(Bytes.add(v1,v2), r.getValue(FAMILY, QUALIFIERS[0])));
4479     assertEquals(0, Bytes.compareTo(Bytes.add(v2,v1), r.getValue(FAMILY, QUALIFIERS[1])));
4480     // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
4481     assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
4482     assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(),
4483         r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp());
4484   }
4485 
4486   @Test
4487   public void testClientPoolRoundRobin() throws IOException {
4488     final TableName tableName = TableName.valueOf("testClientPoolRoundRobin");
4489 
4490     int poolSize = 3;
4491     int numVersions = poolSize * 2;
4492     Configuration conf = TEST_UTIL.getConfiguration();
4493     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4494     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4495 
4496     Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, conf, Integer.MAX_VALUE);
4497 
4498     final long ts = EnvironmentEdgeManager.currentTime();
4499     Get get = new Get(ROW);
4500     get.addColumn(FAMILY, QUALIFIER);
4501     get.setMaxVersions();
4502 
4503     for (int versions = 1; versions <= numVersions; versions++) {
4504       Put put = new Put(ROW);
4505       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4506       table.put(put);
4507 
4508       Result result = table.get(get);
4509       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4510           .get(QUALIFIER);
4511 
4512       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER
4513           + " did not match " + versions, versions, navigableMap.size());
4514       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4515         assertTrue("The value at time " + entry.getKey()
4516             + " did not match what was put",
4517             Bytes.equals(VALUE, entry.getValue()));
4518       }
4519     }
4520   }
4521 
4522   @Ignore ("Flakey: HBASE-8989") @Test
4523   public void testClientPoolThreadLocal() throws IOException {
4524     final TableName tableName = TableName.valueOf("testClientPoolThreadLocal");
4525 
4526     int poolSize = Integer.MAX_VALUE;
4527     int numVersions = 3;
4528     Configuration conf = TEST_UTIL.getConfiguration();
4529     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4530     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4531 
4532     final Table table = TEST_UTIL.createTable(tableName,
4533         new byte[][] { FAMILY }, conf, 3);
4534 
4535     final long ts = EnvironmentEdgeManager.currentTime();
4536     final Get get = new Get(ROW);
4537     get.addColumn(FAMILY, QUALIFIER);
4538     get.setMaxVersions();
4539 
4540     for (int versions = 1; versions <= numVersions; versions++) {
4541       Put put = new Put(ROW);
4542       put.add(FAMILY, QUALIFIER, ts + versions, VALUE);
4543       table.put(put);
4544 
4545       Result result = table.get(get);
4546       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4547           .get(QUALIFIER);
4548 
4549       assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " +
4550         versions + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
4551       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4552         assertTrue("The value at time " + entry.getKey()
4553             + " did not match what was put",
4554             Bytes.equals(VALUE, entry.getValue()));
4555       }
4556     }
4557 
4558     final Object waitLock = new Object();
4559     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4560     final AtomicReference<AssertionError> error = new AtomicReference<AssertionError>(null);
4561     for (int versions = numVersions; versions < numVersions * 2; versions++) {
4562       final int versionsCopy = versions;
4563       executorService.submit(new Callable<Void>() {
4564         @Override
4565         public Void call() {
4566           try {
4567             Put put = new Put(ROW);
4568             put.add(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4569             table.put(put);
4570 
4571             Result result = table.get(get);
4572             NavigableMap<Long, byte[]> navigableMap = result.getMap()
4573                 .get(FAMILY).get(QUALIFIER);
4574 
4575             assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4576                 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
4577                 navigableMap.size());
4578             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4579               assertTrue("The value at time " + entry.getKey()
4580                   + " did not match what was put",
4581                   Bytes.equals(VALUE, entry.getValue()));
4582             }
4583             synchronized (waitLock) {
4584               waitLock.wait();
4585             }
4586           } catch (Exception e) {
4587           } catch (AssertionError e) {
4588             // the error happens in a thread, it won't fail the test,
4589             // need to pass it to the caller for proper handling.
4590             error.set(e);
4591             LOG.error(e);
4592           }
4593 
4594           return null;
4595         }
4596       });
4597     }
4598     synchronized (waitLock) {
4599       waitLock.notifyAll();
4600     }
4601     executorService.shutdownNow();
4602     assertNull(error.get());
4603   }
4604 
4605   @Test
4606   public void testCheckAndPut() throws IOException {
4607     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4608     final byte [] value2 = Bytes.toBytes("abcd");
4609 
4610     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndPut"), FAMILY);
4611     Put put1 = new Put(ROW);
4612     put1.add(FAMILY, QUALIFIER, VALUE);
4613 
4614     // row doesn't exist, so using non-null value should be considered "not match".
4615     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put1);
4616     assertEquals(ok, false);
4617 
4618     // row doesn't exist, so using "null" to check for existence should be considered "match".
4619     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4620     assertEquals(ok, true);
4621 
4622     // row now exists, so using "null" to check for existence should be considered "not match".
4623     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1);
4624     assertEquals(ok, false);
4625 
4626     Put put2 = new Put(ROW);
4627     put2.add(FAMILY, QUALIFIER, value2);
4628 
4629     // row now exists, use the matching value to check
4630     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE, put2);
4631     assertEquals(ok, true);
4632 
4633     Put put3 = new Put(anotherrow);
4634     put3.add(FAMILY, QUALIFIER, VALUE);
4635 
4636     // try to do CheckAndPut on different rows
4637     try {
4638         ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, value2, put3);
4639         fail("trying to check and modify different rows should have failed.");
4640     } catch(Exception e) {}
4641 
4642   }
4643 
4644   @Test
4645   public void testCheckAndPutWithCompareOp() throws IOException {
4646     final byte [] value1 = Bytes.toBytes("aaaa");
4647     final byte [] value2 = Bytes.toBytes("bbbb");
4648     final byte [] value3 = Bytes.toBytes("cccc");
4649     final byte [] value4 = Bytes.toBytes("dddd");
4650 
4651     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndPutWithCompareOp"), FAMILY);
4652 
4653     Put put2 = new Put(ROW);
4654     put2.add(FAMILY, QUALIFIER, value2);
4655 
4656     Put put3 = new Put(ROW);
4657     put3.add(FAMILY, QUALIFIER, value3);
4658 
4659     // row doesn't exist, so using "null" to check for existence should be considered "match".
4660     boolean ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2);
4661     assertEquals(ok, true);
4662 
4663     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4664     // turns out "match"
4665     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, put2);
4666     assertEquals(ok, false);
4667     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, put2);
4668     assertEquals(ok, false);
4669     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, put2);
4670     assertEquals(ok, false);
4671     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, put2);
4672     assertEquals(ok, true);
4673     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, put2);
4674     assertEquals(ok, true);
4675     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, put3);
4676     assertEquals(ok, true);
4677 
4678     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4679     // turns out "match"
4680     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, put3);
4681     assertEquals(ok, false);
4682     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, put3);
4683     assertEquals(ok, false);
4684     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, put3);
4685     assertEquals(ok, false);
4686     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, put3);
4687     assertEquals(ok, true);
4688     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, put3);
4689     assertEquals(ok, true);
4690     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, put2);
4691     assertEquals(ok, true);
4692 
4693     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
4694     // turns out "match"
4695     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, put2);
4696     assertEquals(ok, false);
4697     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, put2);
4698     assertEquals(ok, false);
4699     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, put2);
4700     assertEquals(ok, false);
4701     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, put2);
4702     assertEquals(ok, true);
4703     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, put2);
4704     assertEquals(ok, true);
4705     ok = table.checkAndPut(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, put3);
4706     assertEquals(ok, true);
4707   }
4708 
4709   @Test
4710   public void testCheckAndDeleteWithCompareOp() throws IOException {
4711     final byte [] value1 = Bytes.toBytes("aaaa");
4712     final byte [] value2 = Bytes.toBytes("bbbb");
4713     final byte [] value3 = Bytes.toBytes("cccc");
4714     final byte [] value4 = Bytes.toBytes("dddd");
4715 
4716     Table table = TEST_UTIL.createTable(TableName.valueOf("testCheckAndDeleteWithCompareOp"),
4717         FAMILY);
4718 
4719     Put put2 = new Put(ROW);
4720     put2.add(FAMILY, QUALIFIER, value2);
4721     table.put(put2);
4722 
4723     Put put3 = new Put(ROW);
4724     put3.add(FAMILY, QUALIFIER, value3);
4725 
4726     Delete delete = new Delete(ROW);
4727     delete.deleteColumns(FAMILY, QUALIFIER);
4728 
4729     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4730     // turns out "match"
4731     boolean ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value1, delete);
4732     assertEquals(ok, false);
4733     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value1, delete);
4734     assertEquals(ok, false);
4735     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value1, delete);
4736     assertEquals(ok, false);
4737     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value1, delete);
4738     assertEquals(ok, true);
4739     table.put(put2);
4740     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value1, delete);
4741     assertEquals(ok, true);
4742     table.put(put2);
4743     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value1, delete);
4744     assertEquals(ok, true);
4745 
4746     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4747     // turns out "match"
4748     table.put(put3);
4749     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value4, delete);
4750     assertEquals(ok, false);
4751     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value4, delete);
4752     assertEquals(ok, false);
4753     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value4, delete);
4754     assertEquals(ok, false);
4755     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value4, delete);
4756     assertEquals(ok, true);
4757     table.put(put3);
4758     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value4, delete);
4759     assertEquals(ok, true);
4760     table.put(put3);
4761     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value4, delete);
4762     assertEquals(ok, true);
4763 
4764     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
4765     // turns out "match"
4766     table.put(put2);
4767     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER, value2, delete);
4768     assertEquals(ok, false);
4769     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.NOT_EQUAL, value2, delete);
4770     assertEquals(ok, false);
4771     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS, value2, delete);
4772     assertEquals(ok, false);
4773     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.GREATER_OR_EQUAL, value2, delete);
4774     assertEquals(ok, true);
4775     table.put(put2);
4776     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.LESS_OR_EQUAL, value2, delete);
4777     assertEquals(ok, true);
4778     table.put(put2);
4779     ok = table.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareOp.EQUAL, value2, delete);
4780     assertEquals(ok, true);
4781   }
4782 
4783   /**
4784   * Test ScanMetrics
4785   * @throws Exception
4786   */
4787   @Test
4788   @SuppressWarnings ("unused")
4789   public void testScanMetrics() throws Exception {
4790     TableName TABLENAME = TableName.valueOf("testScanMetrics");
4791 
4792     Configuration conf = TEST_UTIL.getConfiguration();
4793 
4794     // Set up test table:
4795     // Create table:
4796     HTable ht = TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILY);
4797     int numOfRegions = -1;
4798     try (RegionLocator r = ht.getRegionLocator()) {
4799       numOfRegions = r.getStartKeys().length;
4800     }
4801     // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
4802     // scan are forced to hit all the regions.
4803     Put put1 = new Put(Bytes.toBytes("zzz1"));
4804     put1.add(FAMILY, QUALIFIER, VALUE);
4805     Put put2 = new Put(Bytes.toBytes("zzz2"));
4806     put2.add(FAMILY, QUALIFIER, VALUE);
4807     Put put3 = new Put(Bytes.toBytes("zzz3"));
4808     put3.add(FAMILY, QUALIFIER, VALUE);
4809     ht.put(Arrays.asList(put1, put2, put3));
4810 
4811     Scan scan1 = new Scan();
4812     int numRecords = 0;
4813     ResultScanner scanner = ht.getScanner(scan1);
4814     for(Result result : scanner) {
4815       numRecords++;
4816     }
4817     scanner.close();
4818     LOG.info("test data has " + numRecords + " records.");
4819 
4820     // by default, scan metrics collection is turned off
4821     assertEquals(null, scan1.getScanMetrics());
4822 
4823     // turn on scan metrics
4824     Scan scan2 = new Scan();
4825     scan2.setScanMetricsEnabled(true);
4826     scan2.setCaching(numRecords+1);
4827     scanner = ht.getScanner(scan2);
4828     for (Result result : scanner.next(numRecords - 1)) {
4829     }
4830     scanner.close();
4831     // closing the scanner will set the metrics.
4832     assertNotNull(scan2.getScanMetrics());
4833 
4834     // set caching to 1, because metrics are collected in each roundtrip only
4835     scan2 = new Scan();
4836     scan2.setScanMetricsEnabled(true);
4837     scan2.setCaching(1);
4838     scanner = ht.getScanner(scan2);
4839     // per HBASE-5717, this should still collect even if you don't run all the way to
4840     // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
4841     for (Result result : scanner.next(numRecords - 1)) {
4842     }
4843     scanner.close();
4844 
4845     ScanMetrics scanMetrics = scan2.getScanMetrics();
4846     assertEquals("Did not access all the regions in the table", numOfRegions,
4847         scanMetrics.countOfRegions.get());
4848 
4849     // check byte counters
4850     scan2 = new Scan();
4851     scan2.setScanMetricsEnabled(true);
4852     scan2.setCaching(1);
4853     scanner = ht.getScanner(scan2);
4854     int numBytes = 0;
4855     for (Result result : scanner.next(1)) {
4856       for (Cell cell: result.listCells()) {
4857         numBytes += CellUtil.estimatedSerializedSizeOf(cell);
4858       }
4859     }
4860     scanner.close();
4861     scanMetrics = scan2.getScanMetrics();
4862     assertEquals("Did not count the result bytes", numBytes,
4863       scanMetrics.countOfBytesInResults.get());
4864 
4865     // check byte counters on a small scan
4866     scan2 = new Scan();
4867     scan2.setScanMetricsEnabled(true);
4868     scan2.setCaching(1);
4869     scan2.setSmall(true);
4870     scanner = ht.getScanner(scan2);
4871     numBytes = 0;
4872     for (Result result : scanner.next(1)) {
4873       for (Cell cell: result.listCells()) {
4874         numBytes += CellUtil.estimatedSerializedSizeOf(cell);
4875       }
4876     }
4877     scanner.close();
4878     scanMetrics = scan2.getScanMetrics();
4879     assertEquals("Did not count the result bytes", numBytes,
4880       scanMetrics.countOfBytesInResults.get());
4881 
4882     // now, test that the metrics are still collected even if you don't call close, but do
4883     // run past the end of all the records
4884     /** There seems to be a timing issue here.  Comment out for now. Fix when time.
4885     Scan scanWithoutClose = new Scan();
4886     scanWithoutClose.setCaching(1);
4887     scanWithoutClose.setScanMetricsEnabled(true);
4888     ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
4889     for (Result result : scannerWithoutClose.next(numRecords + 1)) {
4890     }
4891     ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
4892     assertEquals("Did not access all the regions in the table", numOfRegions,
4893         scanMetricsWithoutClose.countOfRegions.get());
4894     */
4895 
4896     // finally, test that the metrics are collected correctly if you both run past all the records,
4897     // AND close the scanner
4898     Scan scanWithClose = new Scan();
4899     // make sure we can set caching up to the number of a scanned values
4900     scanWithClose.setCaching(numRecords);
4901     scanWithClose.setScanMetricsEnabled(true);
4902     ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
4903     for (Result result : scannerWithClose.next(numRecords + 1)) {
4904     }
4905     scannerWithClose.close();
4906     ScanMetrics scanMetricsWithClose = getScanMetrics(scanWithClose);
4907     assertEquals("Did not access all the regions in the table", numOfRegions,
4908         scanMetricsWithClose.countOfRegions.get());
4909   }
4910 
4911   private ScanMetrics getScanMetrics(Scan scan) throws Exception {
4912     byte[] serializedMetrics = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
4913     assertTrue("Serialized metrics were not found.", serializedMetrics != null);
4914 
4915     ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(serializedMetrics);
4916 
4917     return scanMetrics;
4918   }
4919 
4920   /**
4921    * Tests that cache on write works all the way up from the client-side.
4922    *
4923    * Performs inserts, flushes, and compactions, verifying changes in the block
4924    * cache along the way.
4925    *
4926    * @throws Exception
4927    */
4928   @Test
4929   public void testCacheOnWriteEvictOnClose() throws Exception {
4930     TableName tableName = TableName.valueOf("testCOWEOCfromClient");
4931     byte [] data = Bytes.toBytes("data");
4932     HTable table = TEST_UTIL.createTable(tableName, FAMILY);
4933     // get the block cache and region
4934     String regionName = table.getRegionLocations().firstKey().getEncodedName();
4935     Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
4936       .getFromOnlineRegions(regionName);
4937     Store store = region.getStores().iterator().next();
4938     CacheConfig cacheConf = store.getCacheConfig();
4939     cacheConf.setCacheDataOnWrite(true);
4940     cacheConf.setEvictOnClose(true);
4941     BlockCache cache = cacheConf.getBlockCache();
4942 
4943     // establish baseline stats
4944     long startBlockCount = cache.getBlockCount();
4945     long startBlockHits = cache.getStats().getHitCount();
4946     long startBlockMiss = cache.getStats().getMissCount();
4947 
4948     // wait till baseline is stable, (minimal 500 ms)
4949     for (int i = 0; i < 5; i++) {
4950       Thread.sleep(100);
4951       if (startBlockCount != cache.getBlockCount()
4952           || startBlockHits != cache.getStats().getHitCount()
4953           || startBlockMiss != cache.getStats().getMissCount()) {
4954         startBlockCount = cache.getBlockCount();
4955         startBlockHits = cache.getStats().getHitCount();
4956         startBlockMiss = cache.getStats().getMissCount();
4957         i = -1;
4958       }
4959     }
4960 
4961     // insert data
4962     Put put = new Put(ROW);
4963     put.add(FAMILY, QUALIFIER, data);
4964     table.put(put);
4965     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
4966     // data was in memstore so don't expect any changes
4967     assertEquals(startBlockCount, cache.getBlockCount());
4968     assertEquals(startBlockHits, cache.getStats().getHitCount());
4969     assertEquals(startBlockMiss, cache.getStats().getMissCount());
4970     // flush the data
4971     System.out.println("Flushing cache");
4972     region.flush(true);
4973     // expect one more block in cache, no change in hits/misses
4974     long expectedBlockCount = startBlockCount + 1;
4975     long expectedBlockHits = startBlockHits;
4976     long expectedBlockMiss = startBlockMiss;
4977     assertEquals(expectedBlockCount, cache.getBlockCount());
4978     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
4979     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4980     // read the data and expect same blocks, one new hit, no misses
4981     assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
4982     assertEquals(expectedBlockCount, cache.getBlockCount());
4983     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
4984     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4985     // insert a second column, read the row, no new blocks, one new hit
4986     byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
4987     byte [] data2 = Bytes.add(data, data);
4988     put = new Put(ROW);
4989     put.add(FAMILY, QUALIFIER2, data2);
4990     table.put(put);
4991     Result r = table.get(new Get(ROW));
4992     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
4993     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
4994     assertEquals(expectedBlockCount, cache.getBlockCount());
4995     assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
4996     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
4997     // flush, one new block
4998     System.out.println("Flushing cache");
4999     region.flush(true);
5000     assertEquals(++expectedBlockCount, cache.getBlockCount());
5001     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5002     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5003     // compact, net minus two blocks, two hits, no misses
5004     System.out.println("Compacting");
5005     assertEquals(2, store.getStorefilesCount());
5006     store.triggerMajorCompaction();
5007     region.compact(true);
5008     waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
5009     assertEquals(1, store.getStorefilesCount());
5010     expectedBlockCount -= 2; // evicted two blocks, cached none
5011     assertEquals(expectedBlockCount, cache.getBlockCount());
5012     expectedBlockHits += 2;
5013     assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5014     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5015     // read the row, this should be a cache miss because we don't cache data
5016     // blocks on compaction
5017     r = table.get(new Get(ROW));
5018     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5019     assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5020     expectedBlockCount += 1; // cached one data block
5021     assertEquals(expectedBlockCount, cache.getBlockCount());
5022     assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5023     assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
5024   }
5025 
5026   private void waitForStoreFileCount(Store store, int count, int timeout)
5027   throws InterruptedException {
5028     long start = System.currentTimeMillis();
5029     while (start + timeout > System.currentTimeMillis() &&
5030         store.getStorefilesCount() != count) {
5031       Thread.sleep(100);
5032     }
5033     System.out.println("start=" + start + ", now=" +
5034         System.currentTimeMillis() + ", cur=" + store.getStorefilesCount());
5035     assertEquals(count, store.getStorefilesCount());
5036   }
5037 
5038   @Test
5039   /**
5040    * Tests the non cached version of getRegionLocator by moving a region.
5041    */
5042   public void testNonCachedGetRegionLocation() throws Exception {
5043     // Test Initialization.
5044     TableName TABLE = TableName.valueOf("testNonCachedGetRegionLocation");
5045     byte [] family1 = Bytes.toBytes("f1");
5046     byte [] family2 = Bytes.toBytes("f2");
5047     try (HTable table = TEST_UTIL.createTable(TABLE, new byte[][] {family1, family2}, 10);
5048         Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration())) {
5049       Map <HRegionInfo, ServerName> regionsMap = table.getRegionLocations();
5050       assertEquals(1, regionsMap.size());
5051       HRegionInfo regionInfo = regionsMap.keySet().iterator().next();
5052       ServerName addrBefore = regionsMap.get(regionInfo);
5053       // Verify region location before move.
5054       HRegionLocation addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5055       HRegionLocation addrNoCache = table.getRegionLocation(regionInfo.getStartKey(),  true);
5056 
5057       assertEquals(addrBefore.getPort(), addrCache.getPort());
5058       assertEquals(addrBefore.getPort(), addrNoCache.getPort());
5059 
5060       ServerName addrAfter = null;
5061       // Now move the region to a different server.
5062       for (int i = 0; i < SLAVES; i++) {
5063         HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
5064         ServerName addr = regionServer.getServerName();
5065         if (addr.getPort() != addrBefore.getPort()) {
5066           admin.move(regionInfo.getEncodedNameAsBytes(),
5067               Bytes.toBytes(addr.toString()));
5068           // Wait for the region to move.
5069           Thread.sleep(5000);
5070           addrAfter = addr;
5071           break;
5072         }
5073       }
5074 
5075       // Verify the region was moved.
5076       addrCache = table.getRegionLocation(regionInfo.getStartKey(), false);
5077       addrNoCache = table.getRegionLocation(regionInfo.getStartKey(), true);
5078       assertNotNull(addrAfter);
5079       assertTrue(addrAfter.getPort() != addrCache.getPort());
5080       assertEquals(addrAfter.getPort(), addrNoCache.getPort());
5081     }
5082   }
5083 
5084   @Test
5085   /**
5086    * Tests getRegionsInRange by creating some regions over which a range of
5087    * keys spans; then changing the key range.
5088    */
5089   public void testGetRegionsInRange() throws Exception {
5090     // Test Initialization.
5091     byte [] startKey = Bytes.toBytes("ddc");
5092     byte [] endKey = Bytes.toBytes("mmm");
5093     TableName TABLE = TableName.valueOf("testGetRegionsInRange");
5094     HTable table = TEST_UTIL.createMultiRegionTable(TABLE, new byte[][] { FAMILY }, 10);
5095     int numOfRegions = -1;
5096     try (RegionLocator r = table.getRegionLocator()) {
5097       numOfRegions = r.getStartKeys().length;
5098     }
5099     assertEquals(26, numOfRegions);
5100 
5101     // Get the regions in this range
5102     List<HRegionLocation> regionsList = table.getRegionsInRange(startKey,
5103       endKey);
5104     assertEquals(10, regionsList.size());
5105 
5106     // Change the start key
5107     startKey = Bytes.toBytes("fff");
5108     regionsList = table.getRegionsInRange(startKey, endKey);
5109     assertEquals(7, regionsList.size());
5110 
5111     // Change the end key
5112     endKey = Bytes.toBytes("nnn");
5113     regionsList = table.getRegionsInRange(startKey, endKey);
5114     assertEquals(8, regionsList.size());
5115 
5116     // Empty start key
5117     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW, endKey);
5118     assertEquals(13, regionsList.size());
5119 
5120     // Empty end key
5121     regionsList = table.getRegionsInRange(startKey, HConstants.EMPTY_END_ROW);
5122     assertEquals(21, regionsList.size());
5123 
5124     // Both start and end keys empty
5125     regionsList = table.getRegionsInRange(HConstants.EMPTY_START_ROW,
5126       HConstants.EMPTY_END_ROW);
5127     assertEquals(26, regionsList.size());
5128 
5129     // Change the end key to somewhere in the last block
5130     endKey = Bytes.toBytes("zzz1");
5131     regionsList = table.getRegionsInRange(startKey, endKey);
5132     assertEquals(21, regionsList.size());
5133 
5134     // Change the start key to somewhere in the first block
5135     startKey = Bytes.toBytes("aac");
5136     regionsList = table.getRegionsInRange(startKey, endKey);
5137     assertEquals(26, regionsList.size());
5138 
5139     // Make start and end key the same
5140     startKey = endKey = Bytes.toBytes("ccc");
5141     regionsList = table.getRegionsInRange(startKey, endKey);
5142     assertEquals(1, regionsList.size());
5143   }
5144 
5145   @Test
5146   public void testJira6912() throws Exception {
5147     TableName TABLE = TableName.valueOf("testJira6912");
5148     Table foo = TEST_UTIL.createTable(TABLE, new byte[][] {FAMILY}, 10);
5149 
5150     List<Put> puts = new ArrayList<Put>();
5151     for (int i=0;i !=100; i++){
5152       Put put = new Put(Bytes.toBytes(i));
5153       put.add(FAMILY, FAMILY, Bytes.toBytes(i));
5154       puts.add(put);
5155     }
5156     foo.put(puts);
5157     // If i comment this out it works
5158     TEST_UTIL.flush();
5159 
5160     Scan scan = new Scan();
5161     scan.setStartRow(Bytes.toBytes(1));
5162     scan.setStopRow(Bytes.toBytes(3));
5163     scan.addColumn(FAMILY, FAMILY);
5164     scan.setFilter(new RowFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(Bytes.toBytes(1))));
5165 
5166     ResultScanner scanner = foo.getScanner(scan);
5167     Result[] bar = scanner.next(100);
5168     assertEquals(1, bar.length);
5169   }
5170 
5171   @Test
5172   public void testScan_NullQualifier() throws IOException {
5173     Table table = TEST_UTIL.createTable(TableName.valueOf("testScan_NullQualifier"), FAMILY);
5174     Put put = new Put(ROW);
5175     put.add(FAMILY, QUALIFIER, VALUE);
5176     table.put(put);
5177 
5178     put = new Put(ROW);
5179     put.add(FAMILY, null, VALUE);
5180     table.put(put);
5181     LOG.info("Row put");
5182 
5183     Scan scan = new Scan();
5184     scan.addColumn(FAMILY, null);
5185 
5186     ResultScanner scanner = table.getScanner(scan);
5187     Result[] bar = scanner.next(100);
5188     assertEquals(1, bar.length);
5189     assertEquals(1, bar[0].size());
5190 
5191     scan = new Scan();
5192     scan.addFamily(FAMILY);
5193 
5194     scanner = table.getScanner(scan);
5195     bar = scanner.next(100);
5196     assertEquals(1, bar.length);
5197     assertEquals(2, bar[0].size());
5198   }
5199 
5200   @Test
5201   public void testNegativeTimestamp() throws IOException {
5202     Table table = TEST_UTIL.createTable(TableName.valueOf("testNegativeTimestamp"), FAMILY);
5203 
5204     try {
5205       Put put = new Put(ROW, -1);
5206       put.add(FAMILY, QUALIFIER, VALUE);
5207       table.put(put);
5208       fail("Negative timestamps should not have been allowed");
5209     } catch (IllegalArgumentException ex) {
5210       assertTrue(ex.getMessage().contains("negative"));
5211     }
5212 
5213     try {
5214       Put put = new Put(ROW);
5215       put.add(FAMILY, QUALIFIER, -1, VALUE);
5216       table.put(put);
5217       fail("Negative timestamps should not have been allowed");
5218     } catch (IllegalArgumentException ex) {
5219       assertTrue(ex.getMessage().contains("negative"));
5220     }
5221 
5222     try {
5223       Delete delete = new Delete(ROW, -1);
5224       table.delete(delete);
5225       fail("Negative timestamps should not have been allowed");
5226     } catch (IllegalArgumentException ex) {
5227       assertTrue(ex.getMessage().contains("negative"));
5228     }
5229 
5230     try {
5231       Delete delete = new Delete(ROW);
5232       delete.deleteFamily(FAMILY, -1);
5233       table.delete(delete);
5234       fail("Negative timestamps should not have been allowed");
5235     } catch (IllegalArgumentException ex) {
5236       assertTrue(ex.getMessage().contains("negative"));
5237     }
5238 
5239     try {
5240       Scan scan = new Scan();
5241       scan.setTimeRange(-1, 1);
5242       table.getScanner(scan);
5243       fail("Negative timestamps should not have been allowed");
5244     } catch (IllegalArgumentException ex) {
5245       assertTrue(ex.getMessage().contains("negative"));
5246     }
5247 
5248     // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
5249     // already has negative timestamps in cluster data, HBase won't be able to handle that
5250     try {
5251       new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
5252     } catch (IllegalArgumentException ex) {
5253       fail("KeyValue SHOULD allow negative timestamps");
5254     }
5255 
5256     table.close();
5257   }
5258 
5259   @Test
5260   public void testIllegalTableDescriptor() throws Exception {
5261     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testIllegalTableDescriptor"));
5262     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
5263 
5264     // create table with 0 families
5265     checkTableIsIllegal(htd);
5266     htd.addFamily(hcd);
5267     checkTableIsLegal(htd);
5268 
5269     htd.setMaxFileSize(1024); // 1K
5270     checkTableIsIllegal(htd);
5271     htd.setMaxFileSize(0);
5272     checkTableIsIllegal(htd);
5273     htd.setMaxFileSize(1024 * 1024 * 1024); // 1G
5274     checkTableIsLegal(htd);
5275 
5276     htd.setMemStoreFlushSize(1024);
5277     checkTableIsIllegal(htd);
5278     htd.setMemStoreFlushSize(0);
5279     checkTableIsIllegal(htd);
5280     htd.setMemStoreFlushSize(128 * 1024 * 1024); // 128M
5281     checkTableIsLegal(htd);
5282 
5283     htd.setRegionSplitPolicyClassName("nonexisting.foo.class");
5284     checkTableIsIllegal(htd);
5285     htd.setRegionSplitPolicyClassName(null);
5286     checkTableIsLegal(htd);
5287 
5288     hcd.setBlocksize(0);
5289     checkTableIsIllegal(htd);
5290     hcd.setBlocksize(1024 * 1024 * 128); // 128M
5291     checkTableIsIllegal(htd);
5292     hcd.setBlocksize(1024);
5293     checkTableIsLegal(htd);
5294 
5295     hcd.setTimeToLive(0);
5296     checkTableIsIllegal(htd);
5297     hcd.setTimeToLive(-1);
5298     checkTableIsIllegal(htd);
5299     hcd.setTimeToLive(1);
5300     checkTableIsLegal(htd);
5301 
5302     hcd.setMinVersions(-1);
5303     checkTableIsIllegal(htd);
5304     hcd.setMinVersions(3);
5305     try {
5306       hcd.setMaxVersions(2);
5307       fail();
5308     } catch (IllegalArgumentException ex) {
5309       // expected
5310       hcd.setMaxVersions(10);
5311     }
5312     checkTableIsLegal(htd);
5313 
5314     // HBASE-13776 Setting illegal versions for HColumnDescriptor
5315     //  does not throw IllegalArgumentException
5316     // finally, minVersions must be less than or equal to maxVersions
5317     hcd.setMaxVersions(4);
5318     hcd.setMinVersions(5);
5319     checkTableIsIllegal(htd);
5320     hcd.setMinVersions(3);
5321 
5322     hcd.setScope(-1);
5323     checkTableIsIllegal(htd);
5324     hcd.setScope(0);
5325     checkTableIsLegal(htd);
5326 
5327     // check the conf settings to disable sanity checks
5328     htd.setMemStoreFlushSize(0);
5329 
5330     // Check that logs warn on invalid table but allow it.
5331     ListAppender listAppender = new ListAppender();
5332     Logger log = Logger.getLogger(HMaster.class);
5333     log.addAppender(listAppender);
5334     log.setLevel(Level.WARN);
5335 
5336     htd.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
5337     checkTableIsLegal(htd);
5338 
5339     assertFalse(listAppender.getMessages().isEmpty());
5340     assertTrue(listAppender.getMessages().get(0).startsWith("MEMSTORE_FLUSHSIZE for table "
5341         + "descriptor or \"hbase.hregion.memstore.flush.size\" (0) is too small, which might "
5342         + "cause very frequent flushing."));
5343 
5344     log.removeAppender(listAppender);
5345   }
5346 
5347   private static class ListAppender extends AppenderSkeleton {
5348     private final List<String> messages = new ArrayList<String>();
5349 
5350     @Override
5351     protected void append(LoggingEvent event) {
5352       messages.add(event.getMessage().toString());
5353     }
5354 
5355     @Override
5356     public void close() {
5357     }
5358 
5359     @Override
5360     public boolean requiresLayout() {
5361       return false;
5362     }
5363 
5364     public List<String> getMessages() {
5365       return messages;
5366     }
5367   }
5368 
5369   private void checkTableIsLegal(HTableDescriptor htd) throws IOException {
5370     Admin admin = TEST_UTIL.getHBaseAdmin();
5371     admin.createTable(htd);
5372     assertTrue(admin.tableExists(htd.getTableName()));
5373     admin.disableTable(htd.getTableName());
5374     admin.deleteTable(htd.getTableName());
5375   }
5376 
5377   private void checkTableIsIllegal(HTableDescriptor htd) throws IOException {
5378     Admin admin = TEST_UTIL.getHBaseAdmin();
5379     try {
5380       admin.createTable(htd);
5381       fail();
5382     } catch(Exception ex) {
5383       // should throw ex
5384     }
5385     assertFalse(admin.tableExists(htd.getTableName()));
5386   }
5387 
5388   @Test
5389   public void testRawScanRespectsVersions() throws Exception {
5390     TableName TABLE = TableName.valueOf("testRawScan");
5391     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5392     byte[] row = Bytes.toBytes("row");
5393 
5394     // put the same row 4 times, with different values
5395     Put p = new Put(row);
5396     p.add(FAMILY, QUALIFIER, 10, VALUE);
5397     table.put(p);
5398     p = new Put(row);
5399     p.add(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
5400     table.put(p);
5401 
5402     p = new Put(row);
5403     p.add(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
5404     table.put(p);
5405 
5406     p = new Put(row);
5407     p.add(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
5408     table.put(p);
5409 
5410     int versions = 4;
5411     Scan s = new Scan(row);
5412     // get all the possible versions
5413     s.setMaxVersions();
5414     s.setRaw(true);
5415 
5416     ResultScanner scanner = table.getScanner(s);
5417     int count = 0;
5418     for (Result r : scanner) {
5419       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5420       count++;
5421     }
5422     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5423       count);
5424     scanner.close();
5425 
5426     // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
5427     // number of versions
5428     versions = 2;
5429     s.setMaxVersions(versions);
5430     scanner = table.getScanner(s);
5431     count = 0;
5432     for (Result r : scanner) {
5433       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5434       count++;
5435     }
5436     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5437       count);
5438     scanner.close();
5439 
5440     // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5441     // to seeing just three
5442     versions = 3;
5443     s.setMaxVersions(versions);
5444     scanner = table.getScanner(s);
5445     count = 0;
5446     for (Result r : scanner) {
5447       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5448       count++;
5449     }
5450     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5451       count);
5452     scanner.close();
5453 
5454     table.close();
5455     TEST_UTIL.deleteTable(TABLE);
5456   }
5457 
5458   @Test
5459   public void testSmallScan() throws Exception {
5460     // Test Initialization.
5461     TableName TABLE = TableName.valueOf("testSmallScan");
5462     Table table = TEST_UTIL.createTable(TABLE, FAMILY);
5463 
5464     // Insert one row each region
5465     int insertNum = 10;
5466     for (int i = 0; i < 10; i++) {
5467       Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5468       put.add(FAMILY, QUALIFIER, VALUE);
5469       table.put(put);
5470     }
5471 
5472     // nomal scan
5473     ResultScanner scanner = table.getScanner(new Scan());
5474     int count = 0;
5475     for (Result r : scanner) {
5476       assertTrue(!r.isEmpty());
5477       count++;
5478     }
5479     assertEquals(insertNum, count);
5480 
5481     // small scan
5482     Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5483     scan.setSmall(true);
5484     scan.setCaching(2);
5485     scanner = table.getScanner(scan);
5486     count = 0;
5487     for (Result r : scanner) {
5488       assertTrue(!r.isEmpty());
5489       count++;
5490     }
5491     assertEquals(insertNum, count);
5492 
5493   }
5494 
5495   @Test
5496   public void testSuperSimpleWithReverseScan() throws Exception {
5497     TableName TABLE = TableName.valueOf("testSuperSimpleWithReverseScan");
5498     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5499     Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
5500     put.add(FAMILY, QUALIFIER, VALUE);
5501     ht.put(put);
5502     put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
5503     put.add(FAMILY, QUALIFIER, VALUE);
5504     ht.put(put);
5505     put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
5506     put.add(FAMILY, QUALIFIER, VALUE);
5507     ht.put(put);
5508     put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
5509     put.add(FAMILY, QUALIFIER, VALUE);
5510     ht.put(put);
5511     put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
5512     put.add(FAMILY, QUALIFIER, VALUE);
5513     ht.put(put);
5514     put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
5515     put.add(FAMILY, QUALIFIER, VALUE);
5516     ht.put(put);
5517     put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
5518     put.add(FAMILY, QUALIFIER, VALUE);
5519     ht.put(put);
5520     put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
5521     put.add(FAMILY, QUALIFIER, VALUE);
5522     ht.put(put);
5523     put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
5524     put.add(FAMILY, QUALIFIER, VALUE);
5525     ht.put(put);
5526     put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
5527     put.add(FAMILY, QUALIFIER, VALUE);
5528     ht.put(put);
5529     Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
5530         Bytes.toBytes("0-b11111-0000000000000000000"));
5531     scan.setReversed(true);
5532     ResultScanner scanner = ht.getScanner(scan);
5533     Result result = scanner.next();
5534     assertTrue(Bytes.equals(result.getRow(),
5535         Bytes.toBytes("0-b11111-0000000000000000008")));
5536     scanner.close();
5537     ht.close();
5538   }
5539 
5540   @Test
5541   public void testFiltersWithReverseScan() throws Exception {
5542     TableName TABLE = TableName.valueOf("testFiltersWithReverseScan");
5543     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5544     byte[][] ROWS = makeN(ROW, 10);
5545     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5546         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5547         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5548         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5549         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5550         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5551         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5552         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5553         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5554         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5555     for (int i = 0; i < 10; i++) {
5556       Put put = new Put(ROWS[i]);
5557       put.add(FAMILY, QUALIFIERS[i], VALUE);
5558       ht.put(put);
5559     }
5560     Scan scan = new Scan();
5561     scan.setReversed(true);
5562     scan.addFamily(FAMILY);
5563     Filter filter = new QualifierFilter(CompareOp.EQUAL,
5564         new RegexStringComparator("col[1-5]"));
5565     scan.setFilter(filter);
5566     ResultScanner scanner = ht.getScanner(scan);
5567     int expectedIndex = 5;
5568     for (Result result : scanner) {
5569       assertEquals(result.size(), 1);
5570       assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
5571       assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
5572           QUALIFIERS[expectedIndex]));
5573       expectedIndex--;
5574     }
5575     assertEquals(expectedIndex, 0);
5576     scanner.close();
5577     ht.close();
5578   }
5579 
5580   @Test
5581   public void testKeyOnlyFilterWithReverseScan() throws Exception {
5582     TableName TABLE = TableName.valueOf("testKeyOnlyFilterWithReverseScan");
5583     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5584     byte[][] ROWS = makeN(ROW, 10);
5585     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5586         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5587         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5588         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5589         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5590         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5591         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5592         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5593         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5594         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5595     for (int i = 0; i < 10; i++) {
5596       Put put = new Put(ROWS[i]);
5597       put.add(FAMILY, QUALIFIERS[i], VALUE);
5598       ht.put(put);
5599     }
5600     Scan scan = new Scan();
5601     scan.setReversed(true);
5602     scan.addFamily(FAMILY);
5603     Filter filter = new KeyOnlyFilter(true);
5604     scan.setFilter(filter);
5605     ResultScanner scanner = ht.getScanner(scan);
5606     int count = 0;
5607     for (Result result : ht.getScanner(scan)) {
5608       assertEquals(result.size(), 1);
5609       assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
5610       assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
5611       count++;
5612     }
5613     assertEquals(count, 10);
5614     scanner.close();
5615     ht.close();
5616   }
5617 
5618   /**
5619    * Test simple table and non-existent row cases.
5620    */
5621   @Test
5622   public void testSimpleMissingWithReverseScan() throws Exception {
5623     TableName TABLE = TableName.valueOf("testSimpleMissingWithReverseScan");
5624     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5625     byte[][] ROWS = makeN(ROW, 4);
5626 
5627     // Try to get a row on an empty table
5628     Scan scan = new Scan();
5629     scan.setReversed(true);
5630     Result result = getSingleScanResult(ht, scan);
5631     assertNullResult(result);
5632 
5633     scan = new Scan(ROWS[0]);
5634     scan.setReversed(true);
5635     result = getSingleScanResult(ht, scan);
5636     assertNullResult(result);
5637 
5638     scan = new Scan(ROWS[0], ROWS[1]);
5639     scan.setReversed(true);
5640     result = getSingleScanResult(ht, scan);
5641     assertNullResult(result);
5642 
5643     scan = new Scan();
5644     scan.setReversed(true);
5645     scan.addFamily(FAMILY);
5646     result = getSingleScanResult(ht, scan);
5647     assertNullResult(result);
5648 
5649     scan = new Scan();
5650     scan.setReversed(true);
5651     scan.addColumn(FAMILY, QUALIFIER);
5652     result = getSingleScanResult(ht, scan);
5653     assertNullResult(result);
5654 
5655     // Insert a row
5656 
5657     Put put = new Put(ROWS[2]);
5658     put.add(FAMILY, QUALIFIER, VALUE);
5659     ht.put(put);
5660 
5661     // Make sure we can scan the row
5662     scan = new Scan();
5663     scan.setReversed(true);
5664     result = getSingleScanResult(ht, scan);
5665     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5666 
5667     scan = new Scan(ROWS[3], ROWS[0]);
5668     scan.setReversed(true);
5669     result = getSingleScanResult(ht, scan);
5670     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5671 
5672     scan = new Scan(ROWS[2], ROWS[1]);
5673     scan.setReversed(true);
5674     result = getSingleScanResult(ht, scan);
5675     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5676 
5677     // Try to scan empty rows around it
5678     // Introduced MemStore#shouldSeekForReverseScan to fix the following
5679     scan = new Scan(ROWS[1]);
5680     scan.setReversed(true);
5681     result = getSingleScanResult(ht, scan);
5682     assertNullResult(result);
5683     ht.close();
5684   }
5685 
5686   @Test
5687   public void testNullWithReverseScan() throws Exception {
5688     TableName TABLE = TableName.valueOf("testNullWithReverseScan");
5689     Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
5690     // Null qualifier (should work)
5691     Put put = new Put(ROW);
5692     put.add(FAMILY, null, VALUE);
5693     ht.put(put);
5694     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5695     Delete delete = new Delete(ROW);
5696     delete.deleteColumns(FAMILY, null);
5697     ht.delete(delete);
5698     // Use a new table
5699     byte[] TABLE2 = Bytes.toBytes("testNull2WithReverseScan");
5700     ht = TEST_UTIL.createTable(TableName.valueOf(TABLE2), FAMILY);
5701     // Empty qualifier, byte[0] instead of null (should work)
5702     put = new Put(ROW);
5703     put.add(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
5704     ht.put(put);
5705     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5706     TEST_UTIL.flush();
5707     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5708     delete = new Delete(ROW);
5709     delete.deleteColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
5710     ht.delete(delete);
5711     // Null value
5712     put = new Put(ROW);
5713     put.add(FAMILY, QUALIFIER, null);
5714     ht.put(put);
5715     Scan scan = new Scan();
5716     scan.setReversed(true);
5717     scan.addColumn(FAMILY, QUALIFIER);
5718     Result result = getSingleScanResult(ht, scan);
5719     assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
5720     ht.close();
5721   }
5722 
5723   @Test
5724   public void testDeletesWithReverseScan() throws Exception {
5725     TableName TABLE = TableName.valueOf("testDeletesWithReverseScan");
5726     byte[][] ROWS = makeNAscii(ROW, 6);
5727     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
5728     byte[][] VALUES = makeN(VALUE, 5);
5729     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
5730     Table ht = TEST_UTIL.createTable(TABLE, FAMILIES, TEST_UTIL.getConfiguration(), 3);
5731 
5732     Put put = new Put(ROW);
5733     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
5734     put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
5735     ht.put(put);
5736 
5737     Delete delete = new Delete(ROW);
5738     delete.deleteFamily(FAMILIES[0], ts[0]);
5739     ht.delete(delete);
5740 
5741     Scan scan = new Scan(ROW);
5742     scan.setReversed(true);
5743     scan.addFamily(FAMILIES[0]);
5744     scan.setMaxVersions(Integer.MAX_VALUE);
5745     Result result = getSingleScanResult(ht, scan);
5746     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] },
5747         new byte[][] { VALUES[1] }, 0, 0);
5748 
5749     // Test delete latest version
5750     put = new Put(ROW);
5751     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
5752     put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
5753     put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
5754     put.add(FAMILIES[0], null, ts[4], VALUES[4]);
5755     put.add(FAMILIES[0], null, ts[2], VALUES[2]);
5756     put.add(FAMILIES[0], null, ts[3], VALUES[3]);
5757     ht.put(put);
5758 
5759     delete = new Delete(ROW);
5760     delete.deleteColumn(FAMILIES[0], QUALIFIER); // ts[4]
5761     ht.delete(delete);
5762 
5763     scan = new Scan(ROW);
5764     scan.setReversed(true);
5765     scan.addColumn(FAMILIES[0], QUALIFIER);
5766     scan.setMaxVersions(Integer.MAX_VALUE);
5767     result = getSingleScanResult(ht, scan);
5768     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
5769         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
5770 
5771     // Test for HBASE-1847
5772     delete = new Delete(ROW);
5773     delete.deleteColumn(FAMILIES[0], null);
5774     ht.delete(delete);
5775 
5776     // Cleanup null qualifier
5777     delete = new Delete(ROW);
5778     delete.deleteColumns(FAMILIES[0], null);
5779     ht.delete(delete);
5780 
5781     // Expected client behavior might be that you can re-put deleted values
5782     // But alas, this is not to be. We can't put them back in either case.
5783 
5784     put = new Put(ROW);
5785     put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
5786     put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
5787     ht.put(put);
5788 
5789     // The Scanner returns the previous values, the expected-naive-unexpected
5790     // behavior
5791 
5792     scan = new Scan(ROW);
5793     scan.setReversed(true);
5794     scan.addFamily(FAMILIES[0]);
5795     scan.setMaxVersions(Integer.MAX_VALUE);
5796     result = getSingleScanResult(ht, scan);
5797     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
5798         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
5799 
5800     // Test deleting an entire family from one row but not the other various
5801     // ways
5802 
5803     put = new Put(ROWS[0]);
5804     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
5805     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
5806     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
5807     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
5808     ht.put(put);
5809 
5810     put = new Put(ROWS[1]);
5811     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
5812     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
5813     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
5814     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
5815     ht.put(put);
5816 
5817     put = new Put(ROWS[2]);
5818     put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
5819     put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
5820     put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
5821     put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
5822     ht.put(put);
5823 
5824     delete = new Delete(ROWS[0]);
5825     delete.deleteFamily(FAMILIES[2]);
5826     ht.delete(delete);
5827 
5828     delete = new Delete(ROWS[1]);
5829     delete.deleteColumns(FAMILIES[1], QUALIFIER);
5830     ht.delete(delete);
5831 
5832     delete = new Delete(ROWS[2]);
5833     delete.deleteColumn(FAMILIES[1], QUALIFIER);
5834     delete.deleteColumn(FAMILIES[1], QUALIFIER);
5835     delete.deleteColumn(FAMILIES[2], QUALIFIER);
5836     ht.delete(delete);
5837 
5838     scan = new Scan(ROWS[0]);
5839     scan.setReversed(true);
5840     scan.addFamily(FAMILIES[1]);
5841     scan.addFamily(FAMILIES[2]);
5842     scan.setMaxVersions(Integer.MAX_VALUE);
5843     result = getSingleScanResult(ht, scan);
5844     assertTrue("Expected 2 keys but received " + result.size(),
5845         result.size() == 2);
5846     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0],
5847         ts[1] }, new byte[][] { VALUES[0], VALUES[1] }, 0, 1);
5848 
5849     scan = new Scan(ROWS[1]);
5850     scan.setReversed(true);
5851     scan.addFamily(FAMILIES[1]);
5852     scan.addFamily(FAMILIES[2]);
5853     scan.setMaxVersions(Integer.MAX_VALUE);
5854     result = getSingleScanResult(ht, scan);
5855     assertTrue("Expected 2 keys but received " + result.size(),
5856         result.size() == 2);
5857 
5858     scan = new Scan(ROWS[2]);
5859     scan.setReversed(true);
5860     scan.addFamily(FAMILIES[1]);
5861     scan.addFamily(FAMILIES[2]);
5862     scan.setMaxVersions(Integer.MAX_VALUE);
5863     result = getSingleScanResult(ht, scan);
5864     assertEquals(1, result.size());
5865     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
5866         new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);
5867 
5868     // Test if we delete the family first in one row (HBASE-1541)
5869 
5870     delete = new Delete(ROWS[3]);
5871     delete.deleteFamily(FAMILIES[1]);
5872     ht.delete(delete);
5873 
5874     put = new Put(ROWS[3]);
5875     put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
5876     ht.put(put);
5877 
5878     put = new Put(ROWS[4]);
5879     put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
5880     put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
5881     ht.put(put);
5882 
5883     scan = new Scan(ROWS[4]);
5884     scan.setReversed(true);
5885     scan.addFamily(FAMILIES[1]);
5886     scan.addFamily(FAMILIES[2]);
5887     scan.setMaxVersions(Integer.MAX_VALUE);
5888     ResultScanner scanner = ht.getScanner(scan);
5889     result = scanner.next();
5890     assertTrue("Expected 2 keys but received " + result.size(),
5891         result.size() == 2);
5892     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[4]));
5893     assertTrue(Bytes.equals(result.raw()[1].getRow(), ROWS[4]));
5894     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[1]));
5895     assertTrue(Bytes.equals(result.raw()[1].getValue(), VALUES[2]));
5896     result = scanner.next();
5897     assertTrue("Expected 1 key but received " + result.size(),
5898         result.size() == 1);
5899     assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[3]));
5900     assertTrue(Bytes.equals(result.raw()[0].getValue(), VALUES[0]));
5901     scanner.close();
5902     ht.close();
5903   }
5904 
5905   /**
5906    * Tests reversed scan under multi regions
5907    */
5908   @Test
5909   public void testReversedScanUnderMultiRegions() throws Exception {
5910     // Test Initialization.
5911     TableName TABLE = TableName.valueOf("testReversedScanUnderMultiRegions");
5912     byte[] maxByteArray = ReversedClientScanner.MAX_BYTE_ARRAY;
5913     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
5914         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
5915         Bytes.toBytes("006"),
5916         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
5917         Bytes.toBytes("007"),
5918         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
5919         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
5920     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
5921     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
5922 
5923     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
5924     // Insert one row each region
5925     int insertNum = splitRows.length;
5926     for (int i = 0; i < insertNum; i++) {
5927       Put put = new Put(splitRows[i]);
5928       put.add(FAMILY, QUALIFIER, VALUE);
5929       table.put(put);
5930     }
5931 
5932     // scan forward
5933     ResultScanner scanner = table.getScanner(new Scan());
5934     int count = 0;
5935     for (Result r : scanner) {
5936       assertTrue(!r.isEmpty());
5937       count++;
5938     }
5939     assertEquals(insertNum, count);
5940 
5941     // scan backward
5942     Scan scan = new Scan();
5943     scan.setReversed(true);
5944     scanner = table.getScanner(scan);
5945     count = 0;
5946     byte[] lastRow = null;
5947     for (Result r : scanner) {
5948       assertTrue(!r.isEmpty());
5949       count++;
5950       byte[] thisRow = r.getRow();
5951       if (lastRow != null) {
5952         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
5953             + ",this row=" + Bytes.toString(thisRow),
5954             Bytes.compareTo(thisRow, lastRow) < 0);
5955       }
5956       lastRow = thisRow;
5957     }
5958     assertEquals(insertNum, count);
5959     table.close();
5960   }
5961 
5962   /**
5963    * Tests reversed scan under multi regions
5964    */
5965   @Test
5966   public void testSmallReversedScanUnderMultiRegions() throws Exception {
5967     // Test Initialization.
5968     TableName TABLE = TableName.valueOf("testSmallReversedScanUnderMultiRegions");
5969     byte[][] splitRows = new byte[][]{
5970         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
5971         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
5972     HTable table = TEST_UTIL.createTable(TABLE, FAMILY, splitRows);
5973     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
5974 
5975     assertEquals(splitRows.length + 1, table.getRegionLocations().size());
5976     for (byte[] splitRow : splitRows) {
5977       Put put = new Put(splitRow);
5978       put.add(FAMILY, QUALIFIER, VALUE);
5979       table.put(put);
5980 
5981       byte[] nextRow = Bytes.copy(splitRow);
5982       nextRow[nextRow.length - 1]++;
5983 
5984       put = new Put(nextRow);
5985       put.add(FAMILY, QUALIFIER, VALUE);
5986       table.put(put);
5987     }
5988 
5989     // scan forward
5990     ResultScanner scanner = table.getScanner(new Scan());
5991     int count = 0;
5992     for (Result r : scanner) {
5993       assertTrue(!r.isEmpty());
5994       count++;
5995     }
5996     assertEquals(12, count);
5997 
5998     reverseScanTest(table, false);
5999     reverseScanTest(table, true);
6000 
6001     table.close();
6002   }
6003 
6004   private void reverseScanTest(Table table, boolean small) throws IOException {
6005     // scan backward
6006     Scan scan = new Scan();
6007     scan.setReversed(true);
6008     ResultScanner scanner = table.getScanner(scan);
6009     int count = 0;
6010     byte[] lastRow = null;
6011     for (Result r : scanner) {
6012       assertTrue(!r.isEmpty());
6013       count++;
6014       byte[] thisRow = r.getRow();
6015       if (lastRow != null) {
6016         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6017             + ",this row=" + Bytes.toString(thisRow),
6018             Bytes.compareTo(thisRow, lastRow) < 0);
6019       }
6020       lastRow = thisRow;
6021     }
6022     assertEquals(12, count);
6023 
6024     scan = new Scan();
6025     scan.setSmall(small);
6026     scan.setReversed(true);
6027     scan.setStartRow(Bytes.toBytes("002"));
6028     scanner = table.getScanner(scan);
6029     count = 0;
6030     lastRow = null;
6031     for (Result r : scanner) {
6032       assertTrue(!r.isEmpty());
6033       count++;
6034       byte[] thisRow = r.getRow();
6035       if (lastRow != null) {
6036         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6037             + ",this row=" + Bytes.toString(thisRow),
6038             Bytes.compareTo(thisRow, lastRow) < 0);
6039       }
6040       lastRow = thisRow;
6041     }
6042     assertEquals(3, count); // 000 001 002
6043 
6044     scan = new Scan();
6045     scan.setSmall(small);
6046     scan.setReversed(true);
6047     scan.setStartRow(Bytes.toBytes("002"));
6048     scan.setStopRow(Bytes.toBytes("000"));
6049     scanner = table.getScanner(scan);
6050     count = 0;
6051     lastRow = null;
6052     for (Result r : scanner) {
6053       assertTrue(!r.isEmpty());
6054       count++;
6055       byte[] thisRow = r.getRow();
6056       if (lastRow != null) {
6057         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6058             + ",this row=" + Bytes.toString(thisRow),
6059             Bytes.compareTo(thisRow, lastRow) < 0);
6060       }
6061       lastRow = thisRow;
6062     }
6063     assertEquals(2, count); // 001 002
6064 
6065     scan = new Scan();
6066     scan.setSmall(small);
6067     scan.setReversed(true);
6068     scan.setStartRow(Bytes.toBytes("001"));
6069     scanner = table.getScanner(scan);
6070     count = 0;
6071     lastRow = null;
6072     for (Result r : scanner) {
6073       assertTrue(!r.isEmpty());
6074       count++;
6075       byte[] thisRow = r.getRow();
6076       if (lastRow != null) {
6077         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6078             + ",this row=" + Bytes.toString(thisRow),
6079             Bytes.compareTo(thisRow, lastRow) < 0);
6080       }
6081       lastRow = thisRow;
6082     }
6083     assertEquals(2, count); // 000 001
6084 
6085     scan = new Scan();
6086     scan.setSmall(small);
6087     scan.setReversed(true);
6088     scan.setStartRow(Bytes.toBytes("000"));
6089     scanner = table.getScanner(scan);
6090     count = 0;
6091     lastRow = null;
6092     for (Result r : scanner) {
6093       assertTrue(!r.isEmpty());
6094       count++;
6095       byte[] thisRow = r.getRow();
6096       if (lastRow != null) {
6097         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6098             + ",this row=" + Bytes.toString(thisRow),
6099             Bytes.compareTo(thisRow, lastRow) < 0);
6100       }
6101       lastRow = thisRow;
6102     }
6103     assertEquals(1, count); // 000
6104 
6105     scan = new Scan();
6106     scan.setSmall(small);
6107     scan.setReversed(true);
6108     scan.setStartRow(Bytes.toBytes("006"));
6109     scan.setStopRow(Bytes.toBytes("002"));
6110     scanner = table.getScanner(scan);
6111     count = 0;
6112     lastRow = null;
6113     for (Result r : scanner) {
6114       assertTrue(!r.isEmpty());
6115       count++;
6116       byte[] thisRow = r.getRow();
6117       if (lastRow != null) {
6118         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6119             + ",this row=" + Bytes.toString(thisRow),
6120             Bytes.compareTo(thisRow, lastRow) < 0);
6121       }
6122       lastRow = thisRow;
6123     }
6124     assertEquals(4, count); // 003 004 005 006
6125   }
6126 
6127   @Test
6128   public void testGetStartEndKeysWithRegionReplicas() throws IOException {
6129     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testGetStartEndKeys"));
6130     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6131     htd.addFamily(fam);
6132     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6133     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
6134     admin.createTable(htd, KEYS);
6135     List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());
6136 
6137     HRegionLocator locator =
6138         (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6139     for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
6140       List<RegionLocations> regionLocations = new ArrayList<RegionLocations>();
6141 
6142       // mock region locations coming from meta with multiple replicas
6143       for (HRegionInfo region : regions) {
6144         HRegionLocation[] arr = new HRegionLocation[regionReplication];
6145         for (int i = 0; i < arr.length; i++) {
6146           arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
6147         }
6148         regionLocations.add(new RegionLocations(arr));
6149       }
6150 
6151       Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
6152 
6153       assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
6154 
6155       for (int i = 0; i < KEYS.length + 1; i++) {
6156         byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
6157         byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
6158         assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
6159         assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
6160       }
6161     }
6162   }
6163 
6164   @Test
6165   public void testFilterAllRecords() throws IOException {
6166     Scan scan = new Scan();
6167     scan.setBatch(1);
6168     scan.setCaching(1);
6169     // Filter out any records
6170     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
6171     try (Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) {
6172       try (ResultScanner s = table.getScanner(scan)) {
6173         assertNull(s.next());
6174       }
6175     }
6176   }
6177 
6178   @Test
6179   public void testRegionCache() throws IOException {
6180     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionCache"));
6181     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6182     htd.addFamily(fam);
6183     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6184     Admin admin = TEST_UTIL.getHBaseAdmin();
6185     admin.createTable(htd, KEYS);
6186     HRegionLocator locator =
6187       (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6188     List<HRegionLocation> results = locator.getAllRegionLocations();
6189     int number = ((ConnectionManager.HConnectionImplementation)admin.getConnection())
6190       .getNumberOfCachedRegionLocations(htd.getTableName());
6191     assertEquals(results.size(), number);
6192   }
6193 }