View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.util.HashSet;
30  import java.util.Set;
31  import java.util.concurrent.Callable;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.client.Admin;
38  import org.apache.hadoop.hbase.client.Get;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Table;
42  import org.apache.hadoop.hbase.master.HMaster;
43  import org.apache.hadoop.hbase.testclassification.MediumTests;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
47  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48  import org.junit.AfterClass;
49  import org.junit.Assert;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Ignore;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  import com.google.common.collect.Sets;
57  
58  @Category(MediumTests.class)
59  public class TestNamespace {
60    protected static final Log LOG = LogFactory.getLog(TestNamespace.class);
61    private static HMaster master;
62    protected final static int NUM_SLAVES_BASE = 4;
63    private static HBaseTestingUtility TEST_UTIL;
64    protected static Admin admin;
65    protected static HBaseCluster cluster;
66    private static ZKNamespaceManager zkNamespaceManager;
67    private String prefix = "TestNamespace";
68  
69  
70    @BeforeClass
71    public static void setUp() throws Exception {
72      TEST_UTIL = new HBaseTestingUtility();
73      TEST_UTIL.getConfiguration().setInt("hbase.namespacejanitor.interval", 5000);
74      TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE);
75      admin = TEST_UTIL.getHBaseAdmin();
76      cluster = TEST_UTIL.getHBaseCluster();
77      master = ((MiniHBaseCluster)cluster).getMaster();
78      zkNamespaceManager =
79          new ZKNamespaceManager(master.getZooKeeper());
80      zkNamespaceManager.start();
81      LOG.info("Done initializing cluster");
82    }
83  
84    @AfterClass
85    public static void tearDown() throws Exception {
86      TEST_UTIL.shutdownMiniCluster();
87    }
88  
89    @Before
90    public void beforeMethod() throws IOException {
91      for (HTableDescriptor desc : admin.listTables(prefix+".*")) {
92        admin.disableTable(desc.getTableName());
93        admin.deleteTable(desc.getTableName());
94      }
95      for (NamespaceDescriptor ns : admin.listNamespaceDescriptors()) {
96        if (ns.getName().startsWith(prefix)) {
97          admin.deleteNamespace(ns.getName());
98        }
99      }
100   }
101 
102   @Test
103   public void verifyReservedNS() throws IOException {
104     //verify existence of reserved namespaces
105     NamespaceDescriptor ns =
106         admin.getNamespaceDescriptor(NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
107     assertNotNull(ns);
108     assertEquals(ns.getName(), NamespaceDescriptor.DEFAULT_NAMESPACE.getName());
109     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
110 
111     ns = admin.getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
112     assertNotNull(ns);
113     assertEquals(ns.getName(), NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
114     assertNotNull(zkNamespaceManager.get(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR));
115 
116     assertEquals(2, admin.listNamespaceDescriptors().length);
117 
118     //verify existence of system tables
119     Set<TableName> systemTables = Sets.newHashSet(
120         TableName.META_TABLE_NAME,
121         TableName.NAMESPACE_TABLE_NAME,
122         TableName.BACKUP_TABLE_NAME);
123     HTableDescriptor[] descs =
124         admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
125     assertTrue(descs.length >= systemTables.size());
126     Set<TableName> tables = new HashSet<>();
127     for (HTableDescriptor desc : descs) {
128       tables.add(desc.getTableName());
129     }
130     assertTrue(tables.containsAll(systemTables));
131     //verify system tables aren't listed
132     assertEquals(0, admin.listTables().length);
133 
134     //Try creating default and system namespaces.
135     boolean exceptionCaught = false;
136     try {
137       admin.createNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE);
138     } catch (IOException exp) {
139       LOG.warn(exp);
140       exceptionCaught = true;
141     } finally {
142       assertTrue(exceptionCaught);
143     }
144 
145     exceptionCaught = false;
146     try {
147       admin.createNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE);
148     } catch (IOException exp) {
149       LOG.warn(exp);
150       exceptionCaught = true;
151     } finally {
152       assertTrue(exceptionCaught);
153     }
154   }
155 
156   @Test
157   public void testDeleteReservedNS() throws Exception {
158     boolean exceptionCaught = false;
159     try {
160       admin.deleteNamespace(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
161     } catch (IOException exp) {
162       LOG.warn(exp);
163       exceptionCaught = true;
164     } finally {
165       assertTrue(exceptionCaught);
166     }
167 
168     try {
169       admin.deleteNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
170     } catch (IOException exp) {
171       LOG.warn(exp);
172       exceptionCaught = true;
173     } finally {
174       assertTrue(exceptionCaught);
175     }
176   }
177 
178   @Test
179   public void createRemoveTest() throws Exception {
180     String testName = "createRemoveTest";
181     String nsName = prefix+"_"+testName;
182     LOG.info(testName);
183 
184     //create namespace and verify
185     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
186     assertEquals(3, admin.listNamespaceDescriptors().length);
187     TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
188       @Override
189       public boolean evaluate() throws Exception {
190         return zkNamespaceManager.list().size() == 3;
191       }
192     });
193     assertNotNull(zkNamespaceManager.get(nsName));
194     //remove namespace and verify
195     admin.deleteNamespace(nsName);
196     assertEquals(2, admin.listNamespaceDescriptors().length);
197     assertEquals(2, zkNamespaceManager.list().size());
198     assertNull(zkNamespaceManager.get(nsName));
199   }
200 
201   @Test
202   public void createDoubleTest() throws IOException, InterruptedException {
203     String testName = "createDoubleTest";
204     String nsName = prefix+"_"+testName;
205     LOG.info(testName);
206 
207     TableName tableName = TableName.valueOf("my_table");
208     TableName tableNameFoo = TableName.valueOf(nsName+":my_table");
209     //create namespace and verify
210     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
211     TEST_UTIL.createTable(tableName, Bytes.toBytes(nsName));
212     TEST_UTIL.createTable(tableNameFoo,Bytes.toBytes(nsName));
213     assertEquals(2, admin.listTables().length);
214     assertNotNull(admin
215         .getTableDescriptor(tableName));
216     assertNotNull(admin
217         .getTableDescriptor(tableNameFoo));
218     //remove namespace and verify
219     admin.disableTable(tableName);
220     admin.deleteTable(tableName);
221     assertEquals(1, admin.listTables().length);
222   }
223 
224   @Test
225   public void createTableTest() throws IOException, InterruptedException {
226     String testName = "createTableTest";
227     String nsName = prefix+"_"+testName;
228     LOG.info(testName);
229 
230     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName+":my_table"));
231     HColumnDescriptor colDesc = new HColumnDescriptor("my_cf");
232     desc.addFamily(colDesc);
233     try {
234       admin.createTable(desc);
235       fail("Expected no namespace exists exception");
236     } catch (NamespaceNotFoundException ex) {
237     }
238     //create table and in new namespace
239     admin.createNamespace(NamespaceDescriptor.create(nsName).build());
240     admin.createTable(desc);
241     TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000);
242     FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
243     assertTrue(fs.exists(
244         new Path(master.getMasterFileSystem().getRootDir(),
245             new Path(HConstants.BASE_NAMESPACE_DIR,
246                 new Path(nsName, desc.getTableName().getQualifierAsString())))));
247     assertEquals(1, admin.listTables().length);
248 
249     //verify non-empty namespace can't be removed
250     try {
251       admin.deleteNamespace(nsName);
252       fail("Expected non-empty namespace constraint exception");
253     } catch (Exception ex) {
254       LOG.info("Caught expected exception: " + ex);
255     }
256 
257     //sanity check try to write and read from table
258     Table table = new HTable(TEST_UTIL.getConfiguration(), desc.getTableName());
259     Put p = new Put(Bytes.toBytes("row1"));
260     p.add(Bytes.toBytes("my_cf"),Bytes.toBytes("my_col"),Bytes.toBytes("value1"));
261     table.put(p);
262     //flush and read from disk to make sure directory changes are working
263     admin.flush(desc.getTableName());
264     Get g = new Get(Bytes.toBytes("row1"));
265     assertTrue(table.exists(g));
266 
267     //normal case of removing namespace
268     TEST_UTIL.deleteTable(desc.getTableName());
269     admin.deleteNamespace(nsName);
270   }
271 
272   @Test
273   public void createTableInDefaultNamespace() throws Exception {
274     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("default_table"));
275     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
276     desc.addFamily(colDesc);
277     admin.createTable(desc);
278     assertTrue(admin.listTables().length == 1);
279     admin.disableTable(desc.getTableName());
280     admin.deleteTable(desc.getTableName());
281   }
282 
283   @Test
284   public void createTableInSystemNamespace() throws Exception {
285     TableName tableName = TableName.valueOf("hbase:createTableInSystemNamespace");
286     HTableDescriptor desc = new HTableDescriptor(tableName);
287     HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
288     desc.addFamily(colDesc);
289     admin.createTable(desc);
290     assertEquals(0, admin.listTables().length);
291     assertTrue(admin.tableExists(tableName));
292     admin.disableTable(desc.getTableName());
293     admin.deleteTable(desc.getTableName());
294   }
295 
296   @Ignore @Test
297   public void testNamespaceJanitor() throws Exception {
298     FileSystem fs = TEST_UTIL.getTestFileSystem();
299 
300     int fsCount = fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
301         HConstants.BASE_NAMESPACE_DIR)).length;
302     Path fakeNSPath =
303         FSUtils.getNamespaceDir(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), "foo");
304     assertTrue(fs.mkdirs(fakeNSPath));
305 
306     String fakeZnode = ZKUtil.joinZNode(ZooKeeperWatcher.namespaceZNode, "foo");
307     int zkCount = ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
308         ZooKeeperWatcher.namespaceZNode).size();
309     ZKUtil.createWithParents(TEST_UTIL.getZooKeeperWatcher(), fakeZnode);
310     Thread.sleep(10000);
311 
312     //verify namespace count is the same and orphan is removed
313     assertFalse(fs.exists(fakeNSPath));
314     assertEquals(fsCount, fs.listStatus(new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
315             HConstants.BASE_NAMESPACE_DIR)).length);
316 
317     assertEquals(-1, ZKUtil.checkExists(TEST_UTIL.getZooKeeperWatcher(), fakeZnode));
318     assertEquals(zkCount,
319         ZKUtil.listChildrenNoWatch(TEST_UTIL.getZooKeeperWatcher(),
320             ZooKeeperWatcher.namespaceZNode).size());
321   }
322 
323   @Test(timeout = 60000)
324   public void testNamespaceOperations() throws IOException {
325     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
326     admin.createNamespace(NamespaceDescriptor.create(prefix + "ns2").build());
327 
328     // create namespace that already exists
329     runWithExpectedException(new Callable<Void>() {
330       @Override
331       public Void call() throws Exception {
332         admin.createNamespace(NamespaceDescriptor.create(prefix + "ns1").build());
333         return null;
334       }
335     }, NamespaceExistException.class);
336 
337     // create a table in non-existing namespace
338     runWithExpectedException(new Callable<Void>() {
339       @Override
340       public Void call() throws Exception {
341         HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("non_existing_namespace", "table1"));
342         htd.addFamily(new HColumnDescriptor("family1"));
343         admin.createTable(htd);
344         return null;
345       }
346     }, NamespaceNotFoundException.class);
347 
348     // get descriptor for existing namespace
349     admin.getNamespaceDescriptor(prefix + "ns1");
350 
351     // get descriptor for non-existing namespace
352     runWithExpectedException(new Callable<NamespaceDescriptor>() {
353       @Override
354       public NamespaceDescriptor call() throws Exception {
355         return admin.getNamespaceDescriptor("non_existing_namespace");
356       }
357     }, NamespaceNotFoundException.class);
358 
359     // delete descriptor for existing namespace
360     admin.deleteNamespace(prefix + "ns2");
361 
362     // delete descriptor for non-existing namespace
363     runWithExpectedException(new Callable<Void>() {
364       @Override
365       public Void call() throws Exception {
366         admin.deleteNamespace("non_existing_namespace");
367         return null;
368       }
369     }, NamespaceNotFoundException.class);
370 
371     // modify namespace descriptor for existing namespace
372     NamespaceDescriptor ns1 = admin.getNamespaceDescriptor(prefix + "ns1");
373     ns1.setConfiguration("foo", "bar");
374     admin.modifyNamespace(ns1);
375 
376     // modify namespace descriptor for non-existing namespace
377     runWithExpectedException(new Callable<Void>() {
378       @Override
379       public Void call() throws Exception {
380         admin.modifyNamespace(NamespaceDescriptor.create("non_existing_namespace").build());
381         return null;
382       }
383     }, NamespaceNotFoundException.class);
384 
385     // get table descriptors for existing namespace
386     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(prefix + "ns1", "table1"));
387     htd.addFamily(new HColumnDescriptor("family1"));
388     admin.createTable(htd);
389     HTableDescriptor[] htds = admin.listTableDescriptorsByNamespace(prefix + "ns1");
390     assertNotNull("Should have not returned null", htds);
391     assertEquals("Should have returned non-empty array", 1, htds.length);
392 
393     // get table descriptors for non-existing namespace
394     runWithExpectedException(new Callable<Void>() {
395       @Override
396       public Void call() throws Exception {
397         admin.listTableDescriptorsByNamespace("non_existing_namespace");
398         return null;
399       }
400     }, NamespaceNotFoundException.class);
401 
402     // get table names for existing namespace
403     TableName[] tableNames = admin.listTableNamesByNamespace(prefix + "ns1");
404     assertNotNull("Should have not returned null", tableNames);
405     assertEquals("Should have returned non-empty array", 1, tableNames.length);
406 
407     // get table names for non-existing namespace
408     runWithExpectedException(new Callable<Void>() {
409       @Override
410       public Void call() throws Exception {
411         admin.listTableNamesByNamespace("non_existing_namespace");
412         return null;
413       }
414     }, NamespaceNotFoundException.class);
415 
416   }
417 
418   private static <V, E> void runWithExpectedException(Callable<V> callable, Class<E> exceptionClass) {
419     try {
420       callable.call();
421     } catch(Exception ex) {
422       Assert.assertEquals(exceptionClass, ex.getClass());
423       return;
424     }
425     fail("Should have thrown exception " + exceptionClass);
426   }
427 
428 }