View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.*;
25  import org.apache.hadoop.hbase.backup.master.BackupController;
26  import org.apache.hadoop.hbase.client.Admin;
27  import org.apache.hadoop.hbase.regionserver.Region;
28  import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29  import org.apache.hadoop.hbase.testclassification.MediumTests;
30  import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32  import org.apache.hadoop.hdfs.MiniDFSCluster;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.ServerLoad;
36  import org.apache.hadoop.hbase.RegionLoad;
37  
38  import java.io.*;
39  import java.util.*;
40  
41  import org.junit.*;
42  import org.junit.experimental.categories.Category;
43  
44  import static org.junit.Assert.assertEquals;
45  import static org.junit.Assert.assertNotNull;
46  import static org.junit.Assert.assertTrue;
47  import static org.junit.Assert.assertFalse;
48  
49  /**
50   * Test coprocessors class loading.
51   */
52  @Category(MediumTests.class)
53  public class TestClassLoading {
54    private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
55    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56  
57    private static MiniDFSCluster cluster;
58  
59    static final TableName tableName = TableName.valueOf("TestClassLoading");
60    static final String cpName1 = "TestCP1";
61    static final String cpName2 = "TestCP2";
62    static final String cpName3 = "TestCP3";
63    static final String cpName4 = "TestCP4";
64    static final String cpName5 = "TestCP5";
65    static final String cpName6 = "TestCP6";
66  
67    private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68    // TOOD: Fix the import of this handler.  It is coming in from a package that is far away.
69    private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70    private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71    private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72    private static Class<?> backupCoprocessor = BackupController.class;
73  
74    private static final String[] regionServerSystemCoprocessors =
75        new String[]{
76        regionServerCoprocessor.getSimpleName()
77    };
78  
79    private static final String[] masterRegionServerSystemCoprocessors = new String[] {
80        regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
81        regionServerCoprocessor.getSimpleName() };
82  
83    @BeforeClass
84    public static void setUpBeforeClass() throws Exception {
85      Configuration conf = TEST_UTIL.getConfiguration();
86  
87      // regionCoprocessor1 will be loaded on all regionservers, since it is
88      // loaded for any tables (user or meta).
89      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
90          regionCoprocessor1.getName());
91  
92      // regionCoprocessor2 will be loaded only on regionservers that serve a
93      // user table region. Therefore, if there are no user tables loaded,
94      // this coprocessor will not be loaded on any regionserver.
95      conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
96          regionCoprocessor2.getName());
97  
98      conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
99          regionServerCoprocessor.getName());
100     conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
101         masterCoprocessor.getName());
102     TEST_UTIL.startMiniCluster(1);
103     cluster = TEST_UTIL.getDFSCluster();
104   }
105 
106   @AfterClass
107   public static void tearDownAfterClass() throws Exception {
108     TEST_UTIL.shutdownMiniCluster();
109   }
110 
111   static File buildCoprocessorJar(String className) throws Exception {
112     String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
113       "public class " + className + " extends BaseRegionObserver {}";
114     return ClassLoaderTestHelper.buildJar(
115       TEST_UTIL.getDataTestDir().toString(), className, code);
116   }
117 
118   @Test
119   // HBASE-3516: Test CP Class loading from HDFS
120   public void testClassLoadingFromHDFS() throws Exception {
121     FileSystem fs = cluster.getFileSystem();
122 
123     File jarFile1 = buildCoprocessorJar(cpName1);
124     File jarFile2 = buildCoprocessorJar(cpName2);
125 
126     // copy the jars into dfs
127     fs.copyFromLocalFile(new Path(jarFile1.getPath()),
128       new Path(fs.getUri().toString() + Path.SEPARATOR));
129     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
130       jarFile1.getName();
131     Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
132     assertTrue("Copy jar file to HDFS failed.",
133       fs.exists(pathOnHDFS1));
134     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
135 
136     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
137         new Path(fs.getUri().toString() + Path.SEPARATOR));
138     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
139       jarFile2.getName();
140     Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
141     assertTrue("Copy jar file to HDFS failed.",
142       fs.exists(pathOnHDFS2));
143     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
144 
145     // create a table that references the coprocessors
146     HTableDescriptor htd = new HTableDescriptor(tableName);
147     htd.addFamily(new HColumnDescriptor("test"));
148       // without configuration values
149     htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
150       "|" + Coprocessor.PRIORITY_USER);
151       // with configuration values
152     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
153       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
154     Admin admin = TEST_UTIL.getHBaseAdmin();
155     if (admin.tableExists(tableName)) {
156       if (admin.isTableEnabled(tableName)) {
157         admin.disableTable(tableName);
158       }
159       admin.deleteTable(tableName);
160     }
161     CoprocessorClassLoader.clearCache();
162     byte[] startKey = {10, 63};
163     byte[] endKey = {12, 43};
164     admin.createTable(htd, startKey, endKey, 4);
165     waitForTable(htd.getTableName());
166 
167     // verify that the coprocessors were loaded
168     boolean foundTableRegion=false;
169     boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
170     Map<Region, Set<ClassLoader>> regionsActiveClassLoaders =
171         new HashMap<Region, Set<ClassLoader>>();
172     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
173     for (Region region:
174         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
175       if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
176         foundTableRegion = true;
177         CoprocessorEnvironment env;
178         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
179         found1 = found1 && (env != null);
180         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
181         found2 = found2 && (env != null);
182         if (env != null) {
183           Configuration conf = env.getConfiguration();
184           found2_k1 = found2_k1 && (conf.get("k1") != null);
185           found2_k2 = found2_k2 && (conf.get("k2") != null);
186           found2_k3 = found2_k3 && (conf.get("k3") != null);
187         } else {
188           found2_k1 = found2_k2 = found2_k3 = false;
189         }
190         regionsActiveClassLoaders
191             .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
192       }
193     }
194 
195     assertTrue("No region was found for table " + tableName, foundTableRegion);
196     assertTrue("Class " + cpName1 + " was missing on a region", found1);
197     assertTrue("Class " + cpName2 + " was missing on a region", found2);
198     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
199     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
200     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
201     // check if CP classloaders are cached
202     assertNotNull(jarFileOnHDFS1 + " was not cached",
203       CoprocessorClassLoader.getIfCached(pathOnHDFS1));
204     assertNotNull(jarFileOnHDFS2 + " was not cached",
205       CoprocessorClassLoader.getIfCached(pathOnHDFS2));
206     //two external jar used, should be one classloader per jar
207     assertEquals("The number of cached classloaders should be equal to the number" +
208       " of external jar files",
209       2, CoprocessorClassLoader.getAllCached().size());
210     //check if region active classloaders are shared across all RS regions
211     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
212       CoprocessorClassLoader.getAllCached());
213     for (Map.Entry<Region, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
214       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
215         + " ClassLoader Cache:" + externalClassLoaders
216         + " Region ClassLoaders:" + regionCP.getValue(),
217         externalClassLoaders.containsAll(regionCP.getValue()));
218     }
219   }
220 
221   private String getLocalPath(File file) {
222     return new Path(file.toURI()).toString();
223   }
224 
225   @Test
226   // HBASE-3516: Test CP Class loading from local file system
227   public void testClassLoadingFromLocalFS() throws Exception {
228     File jarFile = buildCoprocessorJar(cpName3);
229 
230     // create a table that references the jar
231     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
232     htd.addFamily(new HColumnDescriptor("test"));
233     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
234       Coprocessor.PRIORITY_USER);
235     Admin admin = TEST_UTIL.getHBaseAdmin();
236     admin.createTable(htd);
237     waitForTable(htd.getTableName());
238 
239     // verify that the coprocessor was loaded
240     boolean found = false;
241     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
242     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
243       if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
244         found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
245       }
246     }
247     assertTrue("Class " + cpName3 + " was missing on a region", found);
248   }
249 
250   @Test
251   // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
252   public void testPrivateClassLoader() throws Exception {
253     File jarFile = buildCoprocessorJar(cpName4);
254 
255     // create a table that references the jar
256     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
257     htd.addFamily(new HColumnDescriptor("test"));
258     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
259       Coprocessor.PRIORITY_USER);
260     Admin admin = TEST_UTIL.getHBaseAdmin();
261     admin.createTable(htd);
262     waitForTable(htd.getTableName());
263 
264     // verify that the coprocessor was loaded correctly
265     boolean found = false;
266     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
267     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
268       if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) {
269         Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
270         if (cp != null) {
271           found = true;
272           assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
273             cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
274         }
275       }
276     }
277     assertTrue("Class " + cpName4 + " was missing on a region", found);
278   }
279 
280   @Test
281   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
282   // less strict
283   public void testHBase3810() throws Exception {
284     // allowed value pattern: [path] | class name | [priority] | [key values]
285 
286     File jarFile1 = buildCoprocessorJar(cpName1);
287     File jarFile2 = buildCoprocessorJar(cpName2);
288     File jarFile5 = buildCoprocessorJar(cpName5);
289     File jarFile6 = buildCoprocessorJar(cpName6);
290 
291     String cpKey1 = "COPROCESSOR$1";
292     String cpKey2 = " Coprocessor$2 ";
293     String cpKey3 = " coprocessor$03 ";
294 
295     String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
296         Coprocessor.PRIORITY_USER;
297     String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
298     // load from default class loader
299     String cpValue3 =
300         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
301 
302     // create a table that references the jar
303     HTableDescriptor htd = new HTableDescriptor(tableName);
304     htd.addFamily(new HColumnDescriptor("test"));
305 
306     // add 3 coprocessors by setting htd attributes directly.
307     htd.setValue(cpKey1, cpValue1);
308     htd.setValue(cpKey2, cpValue2);
309     htd.setValue(cpKey3, cpValue3);
310 
311     // add 2 coprocessor by using new htd.addCoprocessor() api
312     htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
313         Coprocessor.PRIORITY_USER, null);
314     Map<String, String> kvs = new HashMap<String, String>();
315     kvs.put("k1", "v1");
316     kvs.put("k2", "v2");
317     kvs.put("k3", "v3");
318     htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
319         Coprocessor.PRIORITY_USER, kvs);
320 
321     Admin admin = TEST_UTIL.getHBaseAdmin();
322     if (admin.tableExists(tableName)) {
323       if (admin.isTableEnabled(tableName)) {
324         admin.disableTable(tableName);
325       }
326       admin.deleteTable(tableName);
327     }
328     admin.createTable(htd);
329     waitForTable(htd.getTableName());
330 
331     // verify that the coprocessor was loaded
332     boolean found_2 = false, found_1 = false, found_3 = false,
333         found_5 = false, found_6 = false;
334     boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
335         found6_k4 = false;
336 
337     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
338     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
339       if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
340         found_1 = found_1 ||
341             (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
342         found_2 = found_2 ||
343             (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
344         found_3 = found_3 ||
345             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
346                 != null);
347         found_5 = found_5 ||
348             (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
349 
350         CoprocessorEnvironment env =
351             region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
352         if (env != null) {
353           found_6 = true;
354           Configuration conf = env.getConfiguration();
355           found6_k1 = conf.get("k1") != null;
356           found6_k2 = conf.get("k2") != null;
357           found6_k3 = conf.get("k3") != null;
358         }
359       }
360     }
361 
362     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
363     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
364     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
365     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
366     assertTrue("Class " + cpName6 + " was missing on a region", found_6);
367 
368     assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
369     assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
370     assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
371     assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
372   }
373 
374   @Test
375   public void testClassLoadingFromLibDirInJar() throws Exception {
376     loadingClassFromLibDirInJar("/lib/");
377   }
378 
379   @Test
380   public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
381     loadingClassFromLibDirInJar("lib/");
382   }
383 
384   void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
385     FileSystem fs = cluster.getFileSystem();
386 
387     File innerJarFile1 = buildCoprocessorJar(cpName1);
388     File innerJarFile2 = buildCoprocessorJar(cpName2);
389     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
390 
391     ClassLoaderTestHelper.addJarFilesToJar(
392       outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
393 
394     // copy the jars into dfs
395     fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
396       new Path(fs.getUri().toString() + Path.SEPARATOR));
397     String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
398       outerJarFile.getName();
399     assertTrue("Copy jar file to HDFS failed.",
400       fs.exists(new Path(jarFileOnHDFS)));
401     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
402 
403     // create a table that references the coprocessors
404     HTableDescriptor htd = new HTableDescriptor(tableName);
405     htd.addFamily(new HColumnDescriptor("test"));
406       // without configuration values
407     htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
408       "|" + Coprocessor.PRIORITY_USER);
409       // with configuration values
410     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
411       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
412     Admin admin = TEST_UTIL.getHBaseAdmin();
413     if (admin.tableExists(tableName)) {
414       if (admin.isTableEnabled(tableName)) {
415         admin.disableTable(tableName);
416       }
417       admin.deleteTable(tableName);
418     }
419     admin.createTable(htd);
420     waitForTable(htd.getTableName());
421 
422     // verify that the coprocessors were loaded
423     boolean found1 = false, found2 = false, found2_k1 = false,
424         found2_k2 = false, found2_k3 = false;
425     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
426     for (Region region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
427       if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
428         CoprocessorEnvironment env;
429         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
430         if (env != null) {
431           found1 = true;
432         }
433         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
434         if (env != null) {
435           found2 = true;
436           Configuration conf = env.getConfiguration();
437           found2_k1 = conf.get("k1") != null;
438           found2_k2 = conf.get("k2") != null;
439           found2_k3 = conf.get("k3") != null;
440         }
441       }
442     }
443     assertTrue("Class " + cpName1 + " was missing on a region", found1);
444     assertTrue("Class " + cpName2 + " was missing on a region", found2);
445     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
446     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
447     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
448   }
449 
450   @Test
451   public void testRegionServerCoprocessorsReported() throws Exception {
452     // This was a test for HBASE-4070.
453     // We are removing coprocessors from region load in HBASE-5258.
454     // Therefore, this test now only checks system coprocessors.
455     assertAllRegionServers(null);
456   }
457 
458   /**
459    * return the subset of all regionservers
460    * (actually returns set of ServerLoads)
461    * which host some region in a given table.
462    * used by assertAllRegionServers() below to
463    * test reporting of loaded coprocessors.
464    * @param tableName : given table.
465    * @return subset of all servers.
466    */
467   Map<ServerName, ServerLoad> serversForTable(String tableName) {
468     Map<ServerName, ServerLoad> serverLoadHashMap =
469         new HashMap<ServerName, ServerLoad>();
470     for(Map.Entry<ServerName,ServerLoad> server:
471         TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
472             getOnlineServers().entrySet()) {
473       for( Map.Entry<byte[], RegionLoad> region:
474           server.getValue().getRegionsLoad().entrySet()) {
475         if (region.getValue().getNameAsString().equals(tableName)) {
476           // this server hosts a region of tableName: add this server..
477           serverLoadHashMap.put(server.getKey(),server.getValue());
478           // .. and skip the rest of the regions that it hosts.
479           break;
480         }
481       }
482     }
483     return serverLoadHashMap;
484   }
485 
486   void assertAllRegionServers(String tableName) throws InterruptedException {
487     Map<ServerName, ServerLoad> servers;
488     String[] actualCoprocessors = null;
489     boolean success = false;
490     String[] expectedCoprocessors = regionServerSystemCoprocessors;
491     if (tableName == null) {
492       // if no tableName specified, use all servers.
493       servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
494     } else {
495       servers = serversForTable(tableName);
496     }
497     for (int i = 0; i < 5; i++) {
498       boolean any_failed = false;
499       for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
500         actualCoprocessors = server.getValue().getRsCoprocessors();
501         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
502           LOG.debug("failed comparison: actual: " +
503               Arrays.toString(actualCoprocessors) +
504               " ; expected: " + Arrays.toString(expectedCoprocessors));
505           any_failed = true;
506           expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
507           break;
508         }
509         expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
510       }
511       if (any_failed == false) {
512         success = true;
513         break;
514       }
515       LOG.debug("retrying after failed comparison: " + i);
516       Thread.sleep(1000);
517     }
518     assertTrue(success);
519   }
520 
521   private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
522     if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
523       expectedCoprocessors = masterRegionServerSystemCoprocessors;
524     } else {
525       expectedCoprocessors = regionServerSystemCoprocessors;
526     }
527     return expectedCoprocessors;
528   }
529 
530   @Test
531   public void testMasterCoprocessorsReported() {
532     // HBASE 4070: Improve region server metrics to report loaded coprocessors
533     // to master: verify that the master is reporting the correct set of
534     // loaded coprocessors.
535     final String loadedMasterCoprocessorsVerify =
536         "[" + backupCoprocessor.getSimpleName() + ", " + masterCoprocessor.getSimpleName() + "]";
537     String loadedMasterCoprocessors =
538         java.util.Arrays.toString(
539             TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessors());
540     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
541   }
542 
543   @Test
544   public void testFindCoprocessors() {
545     // HBASE 12277: 
546     CoprocessorHost masterCpHost =
547                              TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost();
548 
549     List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
550 
551     assertTrue(masterObservers != null && masterObservers.size() > 0);
552     assertEquals(masterCoprocessor.getSimpleName(),
553                  masterObservers.get(0).getClass().getSimpleName());
554   }
555 
556   private void waitForTable(TableName name) throws InterruptedException, IOException {
557     // First wait until all regions are online
558     TEST_UTIL.waitTableEnabled(name);
559     // Now wait a bit longer for the coprocessor hosts to load the CPs
560     Thread.sleep(1000);
561   }
562 
563 }
564