1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.*;
25 import org.apache.hadoop.hbase.backup.master.BackupController;
26 import org.apache.hadoop.hbase.client.Admin;
27 import org.apache.hadoop.hbase.regionserver.Region;
28 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29 import org.apache.hadoop.hbase.testclassification.MediumTests;
30 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32 import org.apache.hadoop.hdfs.MiniDFSCluster;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.ServerLoad;
36 import org.apache.hadoop.hbase.RegionLoad;
37
38 import java.io.*;
39 import java.util.*;
40
41 import org.junit.*;
42 import org.junit.experimental.categories.Category;
43
44 import static org.junit.Assert.assertEquals;
45 import static org.junit.Assert.assertNotNull;
46 import static org.junit.Assert.assertTrue;
47 import static org.junit.Assert.assertFalse;
48
49
50
51
52 @Category(MediumTests.class)
53 public class TestClassLoading {
54 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
55 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56
57 private static MiniDFSCluster cluster;
58
59 static final TableName tableName = TableName.valueOf("TestClassLoading");
60 static final String cpName1 = "TestCP1";
61 static final String cpName2 = "TestCP2";
62 static final String cpName3 = "TestCP3";
63 static final String cpName4 = "TestCP4";
64 static final String cpName5 = "TestCP5";
65 static final String cpName6 = "TestCP6";
66
67 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68
69 private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72 private static Class<?> backupCoprocessor = BackupController.class;
73
74 private static final String[] regionServerSystemCoprocessors =
75 new String[]{
76 regionServerCoprocessor.getSimpleName()
77 };
78
79 private static final String[] masterRegionServerSystemCoprocessors = new String[] {
80 regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
81 regionServerCoprocessor.getSimpleName() };
82
83 @BeforeClass
84 public static void setUpBeforeClass() throws Exception {
85 Configuration conf = TEST_UTIL.getConfiguration();
86
87
88
89 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
90 regionCoprocessor1.getName());
91
92
93
94
95 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
96 regionCoprocessor2.getName());
97
98 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
99 regionServerCoprocessor.getName());
100 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
101 masterCoprocessor.getName());
102 TEST_UTIL.startMiniCluster(1);
103 cluster = TEST_UTIL.getDFSCluster();
104 }
105
106 @AfterClass
107 public static void tearDownAfterClass() throws Exception {
108 TEST_UTIL.shutdownMiniCluster();
109 }
110
111 static File buildCoprocessorJar(String className) throws Exception {
112 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
113 "public class " + className + " extends BaseRegionObserver {}";
114 return ClassLoaderTestHelper.buildJar(
115 TEST_UTIL.getDataTestDir().toString(), className, code);
116 }
117
118 @Test
119
120 public void testClassLoadingFromHDFS() throws Exception {
121 FileSystem fs = cluster.getFileSystem();
122
123 File jarFile1 = buildCoprocessorJar(cpName1);
124 File jarFile2 = buildCoprocessorJar(cpName2);
125
126
127 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
128 new Path(fs.getUri().toString() + Path.SEPARATOR));
129 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
130 jarFile1.getName();
131 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
132 assertTrue("Copy jar file to HDFS failed.",
133 fs.exists(pathOnHDFS1));
134 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
135
136 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
137 new Path(fs.getUri().toString() + Path.SEPARATOR));
138 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
139 jarFile2.getName();
140 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
141 assertTrue("Copy jar file to HDFS failed.",
142 fs.exists(pathOnHDFS2));
143 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
144
145
146 HTableDescriptor htd = new HTableDescriptor(tableName);
147 htd.addFamily(new HColumnDescriptor("test"));
148
149 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
150 "|" + Coprocessor.PRIORITY_USER);
151
152 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
153 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
154 Admin admin = TEST_UTIL.getHBaseAdmin();
155 if (admin.tableExists(tableName)) {
156 if (admin.isTableEnabled(tableName)) {
157 admin.disableTable(tableName);
158 }
159 admin.deleteTable(tableName);
160 }
161 CoprocessorClassLoader.clearCache();
162 byte[] startKey = {10, 63};
163 byte[] endKey = {12, 43};
164 admin.createTable(htd, startKey, endKey, 4);
165 waitForTable(htd.getTableName());
166
167
168 boolean foundTableRegion=false;
169 boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
170 Map<Region, Set<ClassLoader>> regionsActiveClassLoaders =
171 new HashMap<Region, Set<ClassLoader>>();
172 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
173 for (Region region:
174 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
175 if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
176 foundTableRegion = true;
177 CoprocessorEnvironment env;
178 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
179 found1 = found1 && (env != null);
180 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
181 found2 = found2 && (env != null);
182 if (env != null) {
183 Configuration conf = env.getConfiguration();
184 found2_k1 = found2_k1 && (conf.get("k1") != null);
185 found2_k2 = found2_k2 && (conf.get("k2") != null);
186 found2_k3 = found2_k3 && (conf.get("k3") != null);
187 } else {
188 found2_k1 = found2_k2 = found2_k3 = false;
189 }
190 regionsActiveClassLoaders
191 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
192 }
193 }
194
195 assertTrue("No region was found for table " + tableName, foundTableRegion);
196 assertTrue("Class " + cpName1 + " was missing on a region", found1);
197 assertTrue("Class " + cpName2 + " was missing on a region", found2);
198 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
199 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
200 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
201
202 assertNotNull(jarFileOnHDFS1 + " was not cached",
203 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
204 assertNotNull(jarFileOnHDFS2 + " was not cached",
205 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
206
207 assertEquals("The number of cached classloaders should be equal to the number" +
208 " of external jar files",
209 2, CoprocessorClassLoader.getAllCached().size());
210
211 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
212 CoprocessorClassLoader.getAllCached());
213 for (Map.Entry<Region, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
214 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
215 + " ClassLoader Cache:" + externalClassLoaders
216 + " Region ClassLoaders:" + regionCP.getValue(),
217 externalClassLoaders.containsAll(regionCP.getValue()));
218 }
219 }
220
221 private String getLocalPath(File file) {
222 return new Path(file.toURI()).toString();
223 }
224
225 @Test
226
227 public void testClassLoadingFromLocalFS() throws Exception {
228 File jarFile = buildCoprocessorJar(cpName3);
229
230
231 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
232 htd.addFamily(new HColumnDescriptor("test"));
233 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
234 Coprocessor.PRIORITY_USER);
235 Admin admin = TEST_UTIL.getHBaseAdmin();
236 admin.createTable(htd);
237 waitForTable(htd.getTableName());
238
239
240 boolean found = false;
241 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
242 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
243 if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
244 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
245 }
246 }
247 assertTrue("Class " + cpName3 + " was missing on a region", found);
248 }
249
250 @Test
251
252 public void testPrivateClassLoader() throws Exception {
253 File jarFile = buildCoprocessorJar(cpName4);
254
255
256 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
257 htd.addFamily(new HColumnDescriptor("test"));
258 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
259 Coprocessor.PRIORITY_USER);
260 Admin admin = TEST_UTIL.getHBaseAdmin();
261 admin.createTable(htd);
262 waitForTable(htd.getTableName());
263
264
265 boolean found = false;
266 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
267 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
268 if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) {
269 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
270 if (cp != null) {
271 found = true;
272 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
273 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
274 }
275 }
276 }
277 assertTrue("Class " + cpName4 + " was missing on a region", found);
278 }
279
280 @Test
281
282
283 public void testHBase3810() throws Exception {
284
285
286 File jarFile1 = buildCoprocessorJar(cpName1);
287 File jarFile2 = buildCoprocessorJar(cpName2);
288 File jarFile5 = buildCoprocessorJar(cpName5);
289 File jarFile6 = buildCoprocessorJar(cpName6);
290
291 String cpKey1 = "COPROCESSOR$1";
292 String cpKey2 = " Coprocessor$2 ";
293 String cpKey3 = " coprocessor$03 ";
294
295 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
296 Coprocessor.PRIORITY_USER;
297 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
298
299 String cpValue3 =
300 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
301
302
303 HTableDescriptor htd = new HTableDescriptor(tableName);
304 htd.addFamily(new HColumnDescriptor("test"));
305
306
307 htd.setValue(cpKey1, cpValue1);
308 htd.setValue(cpKey2, cpValue2);
309 htd.setValue(cpKey3, cpValue3);
310
311
312 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
313 Coprocessor.PRIORITY_USER, null);
314 Map<String, String> kvs = new HashMap<String, String>();
315 kvs.put("k1", "v1");
316 kvs.put("k2", "v2");
317 kvs.put("k3", "v3");
318 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
319 Coprocessor.PRIORITY_USER, kvs);
320
321 Admin admin = TEST_UTIL.getHBaseAdmin();
322 if (admin.tableExists(tableName)) {
323 if (admin.isTableEnabled(tableName)) {
324 admin.disableTable(tableName);
325 }
326 admin.deleteTable(tableName);
327 }
328 admin.createTable(htd);
329 waitForTable(htd.getTableName());
330
331
332 boolean found_2 = false, found_1 = false, found_3 = false,
333 found_5 = false, found_6 = false;
334 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
335 found6_k4 = false;
336
337 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
338 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
339 if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
340 found_1 = found_1 ||
341 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
342 found_2 = found_2 ||
343 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
344 found_3 = found_3 ||
345 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
346 != null);
347 found_5 = found_5 ||
348 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
349
350 CoprocessorEnvironment env =
351 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
352 if (env != null) {
353 found_6 = true;
354 Configuration conf = env.getConfiguration();
355 found6_k1 = conf.get("k1") != null;
356 found6_k2 = conf.get("k2") != null;
357 found6_k3 = conf.get("k3") != null;
358 }
359 }
360 }
361
362 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
363 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
364 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
365 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
366 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
367
368 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
369 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
370 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
371 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
372 }
373
374 @Test
375 public void testClassLoadingFromLibDirInJar() throws Exception {
376 loadingClassFromLibDirInJar("/lib/");
377 }
378
379 @Test
380 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
381 loadingClassFromLibDirInJar("lib/");
382 }
383
384 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
385 FileSystem fs = cluster.getFileSystem();
386
387 File innerJarFile1 = buildCoprocessorJar(cpName1);
388 File innerJarFile2 = buildCoprocessorJar(cpName2);
389 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
390
391 ClassLoaderTestHelper.addJarFilesToJar(
392 outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
393
394
395 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
396 new Path(fs.getUri().toString() + Path.SEPARATOR));
397 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
398 outerJarFile.getName();
399 assertTrue("Copy jar file to HDFS failed.",
400 fs.exists(new Path(jarFileOnHDFS)));
401 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
402
403
404 HTableDescriptor htd = new HTableDescriptor(tableName);
405 htd.addFamily(new HColumnDescriptor("test"));
406
407 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
408 "|" + Coprocessor.PRIORITY_USER);
409
410 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
411 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
412 Admin admin = TEST_UTIL.getHBaseAdmin();
413 if (admin.tableExists(tableName)) {
414 if (admin.isTableEnabled(tableName)) {
415 admin.disableTable(tableName);
416 }
417 admin.deleteTable(tableName);
418 }
419 admin.createTable(htd);
420 waitForTable(htd.getTableName());
421
422
423 boolean found1 = false, found2 = false, found2_k1 = false,
424 found2_k2 = false, found2_k3 = false;
425 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
426 for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
427 if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
428 CoprocessorEnvironment env;
429 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
430 if (env != null) {
431 found1 = true;
432 }
433 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
434 if (env != null) {
435 found2 = true;
436 Configuration conf = env.getConfiguration();
437 found2_k1 = conf.get("k1") != null;
438 found2_k2 = conf.get("k2") != null;
439 found2_k3 = conf.get("k3") != null;
440 }
441 }
442 }
443 assertTrue("Class " + cpName1 + " was missing on a region", found1);
444 assertTrue("Class " + cpName2 + " was missing on a region", found2);
445 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
446 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
447 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
448 }
449
450 @Test
451 public void testRegionServerCoprocessorsReported() throws Exception {
452
453
454
455 assertAllRegionServers(null);
456 }
457
458
459
460
461
462
463
464
465
466
467 Map<ServerName, ServerLoad> serversForTable(String tableName) {
468 Map<ServerName, ServerLoad> serverLoadHashMap =
469 new HashMap<ServerName, ServerLoad>();
470 for(Map.Entry<ServerName,ServerLoad> server:
471 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
472 getOnlineServers().entrySet()) {
473 for( Map.Entry<byte[], RegionLoad> region:
474 server.getValue().getRegionsLoad().entrySet()) {
475 if (region.getValue().getNameAsString().equals(tableName)) {
476
477 serverLoadHashMap.put(server.getKey(),server.getValue());
478
479 break;
480 }
481 }
482 }
483 return serverLoadHashMap;
484 }
485
486 void assertAllRegionServers(String tableName) throws InterruptedException {
487 Map<ServerName, ServerLoad> servers;
488 String[] actualCoprocessors = null;
489 boolean success = false;
490 String[] expectedCoprocessors = regionServerSystemCoprocessors;
491 if (tableName == null) {
492
493 servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
494 } else {
495 servers = serversForTable(tableName);
496 }
497 for (int i = 0; i < 5; i++) {
498 boolean any_failed = false;
499 for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
500 actualCoprocessors = server.getValue().getRsCoprocessors();
501 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
502 LOG.debug("failed comparison: actual: " +
503 Arrays.toString(actualCoprocessors) +
504 " ; expected: " + Arrays.toString(expectedCoprocessors));
505 any_failed = true;
506 expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
507 break;
508 }
509 expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
510 }
511 if (any_failed == false) {
512 success = true;
513 break;
514 }
515 LOG.debug("retrying after failed comparison: " + i);
516 Thread.sleep(1000);
517 }
518 assertTrue(success);
519 }
520
521 private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
522 if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
523 expectedCoprocessors = masterRegionServerSystemCoprocessors;
524 } else {
525 expectedCoprocessors = regionServerSystemCoprocessors;
526 }
527 return expectedCoprocessors;
528 }
529
530 @Test
531 public void testMasterCoprocessorsReported() {
532
533
534
535 final String loadedMasterCoprocessorsVerify =
536 "[" + backupCoprocessor.getSimpleName() + ", " + masterCoprocessor.getSimpleName() + "]";
537 String loadedMasterCoprocessors =
538 java.util.Arrays.toString(
539 TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessors());
540 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
541 }
542
543 @Test
544 public void testFindCoprocessors() {
545
546 CoprocessorHost masterCpHost =
547 TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost();
548
549 List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
550
551 assertTrue(masterObservers != null && masterObservers.size() > 0);
552 assertEquals(masterCoprocessor.getSimpleName(),
553 masterObservers.get(0).getClass().getSimpleName());
554 }
555
556 private void waitForTable(TableName name) throws InterruptedException, IOException {
557
558 TEST_UTIL.waitTableEnabled(name);
559
560 Thread.sleep(1000);
561 }
562
563 }
564