View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertTrue;
25  import static org.mockito.Mockito.doReturn;
26  import static org.mockito.Mockito.spy;
27  
28  import java.io.IOException;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.SortedMap;
32  import java.util.TreeMap;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.hbase.ChoreService;
42  import org.apache.hadoop.hbase.CoordinatedStateManager;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.MetaMockingUtil;
49  import org.apache.hadoop.hbase.NamespaceDescriptor;
50  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
51  import org.apache.hadoop.hbase.ProcedureInfo;
52  import org.apache.hadoop.hbase.Server;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.testclassification.SmallTests;
55  import org.apache.hadoop.hbase.TableDescriptors;
56  import org.apache.hadoop.hbase.TableName;
57  import org.apache.hadoop.hbase.TableStateManager;
58  import org.apache.hadoop.hbase.backup.BackupType;
59  import org.apache.hadoop.hbase.client.ClusterConnection;
60  import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
61  import org.apache.hadoop.hbase.client.Result;
62  import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
63  import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
64  import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
65  import org.apache.hadoop.hbase.executor.ExecutorService;
66  import org.apache.hadoop.hbase.io.Reference;
67  import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
68  import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
69  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
70  import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
71  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
72  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
74  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
75  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
76  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
77  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
78  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
79  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
80  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
81  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
82  import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
83  import org.apache.hadoop.hbase.regionserver.HStore;
84  import org.apache.hadoop.hbase.util.Bytes;
85  import org.apache.hadoop.hbase.util.FSUtils;
86  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
87  import org.apache.hadoop.hbase.util.Pair;
88  import org.apache.hadoop.hbase.util.Triple;
89  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
90  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
91  import org.junit.Test;
92  import org.junit.experimental.categories.Category;
93  import org.mockito.Mockito;
94  import org.mockito.invocation.InvocationOnMock;
95  import org.mockito.stubbing.Answer;
96  
97  import com.google.protobuf.RpcController;
98  import com.google.protobuf.Service;
99  import com.google.protobuf.ServiceException;
100 
101 @Category(SmallTests.class)
102 public class TestCatalogJanitor {
103   private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
104 
105   /**
106    * Pseudo server for below tests.
107    * Be sure to call stop on the way out else could leave some mess around.
108    */
109   class MockServer implements Server {
110     private final ClusterConnection connection;
111     private final Configuration c;
112 
113     MockServer(final HBaseTestingUtility htu)
114     throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
115       this.c = htu.getConfiguration();
116       ClientProtos.ClientService.BlockingInterface ri =
117         Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
118       MutateResponse.Builder builder = MutateResponse.newBuilder();
119       builder.setProcessed(true);
120       try {
121         Mockito.when(ri.mutate(
122           (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
123             thenReturn(builder.build());
124       } catch (ServiceException se) {
125         throw ProtobufUtil.getRemoteException(se);
126       }
127       try {
128         Mockito.when(ri.multi(
129           (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
130             thenAnswer(new Answer<MultiResponse>() {
131               @Override
132               public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
133                 return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
134               }
135             });
136       } catch (ServiceException se) {
137         throw ProtobufUtil.getRemoteException(se);
138       }
139       // Mock an HConnection and a AdminProtocol implementation.  Have the
140       // HConnection return the HRI.  Have the HRI return a few mocked up responses
141       // to make our test work.
142       this.connection =
143         HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
144           Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
145             ServerName.valueOf("example.org,12345,6789"),
146           HRegionInfo.FIRST_META_REGIONINFO);
147       // Set hbase.rootdir into test dir.
148       FileSystem fs = FileSystem.get(this.c);
149       Path rootdir = FSUtils.getRootDir(this.c);
150       FSUtils.setRootDir(this.c, rootdir);
151       AdminProtos.AdminService.BlockingInterface hri =
152         Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
153     }
154 
155     @Override
156     public ClusterConnection getConnection() {
157       return this.connection;
158     }
159 
160     @Override
161     public MetaTableLocator getMetaTableLocator() {
162       return null;
163     }
164 
165     @Override
166     public Configuration getConfiguration() {
167       return this.c;
168     }
169 
170     @Override
171     public ServerName getServerName() {
172       return ServerName.valueOf("mockserver.example.org", 1234, -1L);
173     }
174 
175     @Override
176     public ZooKeeperWatcher getZooKeeper() {
177       return null;
178     }
179 
180     @Override
181     public CoordinatedStateManager getCoordinatedStateManager() {
182       BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
183       SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
184       Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
185       SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
186       Mockito.when(c.getDetails()).thenReturn(d);
187       return m;
188     }
189 
190     @Override
191     public void abort(String why, Throwable e) {
192       //no-op
193     }
194 
195     @Override
196     public boolean isAborted() {
197       return false;
198     }
199 
200     @Override
201     public boolean isStopped() {
202       return false;
203     }
204 
205     @Override
206     public void stop(String why) {
207     }
208 
209     @Override
210     public ChoreService getChoreService() {
211       return null;
212     }
213   }
214 
215   /**
216    * Mock MasterServices for tests below.
217    */
218   class MockMasterServices implements MasterServices {
219     private final MasterFileSystem mfs;
220     private final AssignmentManager asm;
221 
222     MockMasterServices(final Server server) throws IOException {
223       this.mfs = new MasterFileSystem(server, this);
224       this.asm = Mockito.mock(AssignmentManager.class);
225     }
226 
227     @Override
228     public void checkTableModifiable(TableName tableName) throws IOException {
229       //no-op
230     }
231 
232     @Override
233     public boolean isMasterProcedureExecutorEnabled() {
234       return true;
235     }
236 
237     @Override
238     public long createTable(HTableDescriptor desc, byte[][] splitKeys)
239         throws IOException {
240       // no-op
241       return -1;
242     }
243 
244     @Override
245     public SnapshotManager getSnapshotManager() {
246       return null;
247     }
248 
249     @Override
250     public MasterProcedureManagerHost getMasterProcedureManagerHost() {
251       return null;
252     }
253 
254     @Override
255     public AssignmentManager getAssignmentManager() {
256       return this.asm;
257     }
258 
259     @Override
260     public ExecutorService getExecutorService() {
261       return null;
262     }
263 
264     @Override
265     public ChoreService getChoreService() {
266       return null;
267     }
268 
269     @Override
270     public MasterFileSystem getMasterFileSystem() {
271       return this.mfs;
272     }
273 
274     @Override
275     public MasterCoprocessorHost getMasterCoprocessorHost() {
276       return null;
277     }
278 
279     @Override
280     public MasterQuotaManager getMasterQuotaManager() {
281       return null;
282     }
283 
284     @Override
285     public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
286       return null;
287     }
288 
289     @Override
290     public ServerManager getServerManager() {
291       return null;
292     }
293 
294     @Override
295     public ZooKeeperWatcher getZooKeeper() {
296       return null;
297     }
298 
299     @Override
300     public CoordinatedStateManager getCoordinatedStateManager() {
301       return null;
302     }
303 
304     @Override
305     public MetaTableLocator getMetaTableLocator() {
306       return null;
307     }
308 
309     @Override
310     public ClusterConnection getConnection() {
311       return null;
312     }
313 
314     @Override
315     public Configuration getConfiguration() {
316       return mfs.conf;
317     }
318 
319     @Override
320     public ServerName getServerName() {
321       return null;
322     }
323 
324     @Override
325     public void abort(String why, Throwable e) {
326       //no-op
327     }
328 
329     @Override
330     public boolean isAborted() {
331       return false;
332     }
333 
334     private boolean stopped = false;
335 
336     @Override
337     public void stop(String why) {
338       stopped = true;
339     }
340 
341     @Override
342     public boolean isStopped() {
343       return stopped;
344     }
345 
346     @Override
347     public TableDescriptors getTableDescriptors() {
348       return new TableDescriptors() {
349         @Override
350         public HTableDescriptor remove(TableName tablename) throws IOException {
351           // TODO Auto-generated method stub
352           return null;
353         }
354 
355         @Override
356         public Map<String, HTableDescriptor> getAll() throws IOException {
357           // TODO Auto-generated method stub
358           return null;
359         }
360 
361         @Override
362         public HTableDescriptor get(TableName tablename)
363         throws IOException {
364           return createHTableDescriptor();
365         }
366 
367         @Override
368         public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
369           return null;
370         }
371 
372         @Override
373         public void add(HTableDescriptor htd) throws IOException {
374           // TODO Auto-generated method stub
375 
376         }
377         @Override
378         public void setCacheOn() throws IOException {
379         }
380 
381         @Override
382         public void setCacheOff() throws IOException {
383         }
384       };
385     }
386 
387     @Override
388     public boolean isServerShutdownHandlerEnabled() {
389       return true;
390     }
391 
392     @Override
393     public boolean registerService(Service instance) {
394       return false;
395     }
396 
397     @Override
398     public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
399       //To change body of implemented methods use File | Settings | File Templates.
400     }
401 
402     @Override
403     public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
404       //To change body of implemented methods use File | Settings | File Templates.
405     }
406 
407     @Override
408     public void deleteNamespace(String name) throws IOException {
409       //To change body of implemented methods use File | Settings | File Templates.
410     }
411 
412     @Override
413     public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
414       return null;  //To change body of implemented methods use File | Settings | File Templates.
415     }
416 
417     @Override
418     public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
419       return null;  //To change body of implemented methods use File | Settings | File Templates.
420     }
421 
422     @Override
423     public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
424         throws IOException {
425       return false;  //To change body of implemented methods use File | Settings | File Templates.
426     }
427 
428     @Override
429     public List<ProcedureInfo> listProcedures() throws IOException {
430       return null;  //To change body of implemented methods use File | Settings | File Templates.
431     }
432 
433     @Override
434     public Pair<Long, String> backupTables(
435         final BackupType type,
436         final List<TableName> tableList,
437         final String targetRootDir, final int workers,
438         final long bandwidth) throws IOException {
439       return null;
440     }
441 
442 
443     @Override
444     public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
445       return null;  //To change body of implemented methods use File | Settings | File Templates.
446     }
447 
448     @Override
449     public List<TableName> listTableNamesByNamespace(String name) throws IOException {
450       return null;
451     }
452 
453     @Override
454     public long deleteTable(TableName tableName) throws IOException {
455       return -1;
456     }
457     @Override
458     public LoadBalancer getLoadBalancer() {
459       return null;
460     }
461 
462     @Override
463     public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException { }
464 
465 
466     @Override
467     public void modifyTable(TableName tableName, HTableDescriptor descriptor)
468         throws IOException { }
469 
470     @Override
471     public long enableTable(TableName tableName) throws IOException {
472       return -1;
473     }
474 
475     @Override
476     public long disableTable(TableName tableName) throws IOException {
477       return -1;
478     }
479 
480     @Override
481     public void addColumn(TableName tableName, HColumnDescriptor column)
482         throws IOException { }
483 
484     @Override
485     public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
486         throws IOException { }
487 
488     @Override
489     public void deleteColumn(TableName tableName, byte[] columnName)
490         throws IOException { }
491 
492     @Override
493     public TableLockManager getTableLockManager() {
494       return null;
495     }
496 
497     @Override
498     public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
499         boolean forcible) throws IOException {
500     }
501 
502     @Override
503     public boolean isInitialized() {
504       // Auto-generated method stub
505       return false;
506     }
507 
508     @Override
509     public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
510       // Auto-generated method stub
511       return 0;
512     }
513 
514     @Override
515     public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
516       // Auto-generated method stub
517       return 0;
518     }
519 
520     @Override
521     public TableStateManager getTableStateManager() {
522       // TODO Auto-generated method stub
523       return null;
524     }
525   }
526 
527   @Test
528   public void testCleanParent() throws IOException, InterruptedException {
529     HBaseTestingUtility htu = new HBaseTestingUtility();
530     setRootDirAndCleanIt(htu, "testCleanParent");
531     Server server = new MockServer(htu);
532     try {
533       MasterServices services = new MockMasterServices(server);
534       CatalogJanitor janitor = new CatalogJanitor(server, services);
535       // Create regions.
536       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
537       htd.addFamily(new HColumnDescriptor("f"));
538       HRegionInfo parent =
539         new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
540             Bytes.toBytes("eee"));
541       HRegionInfo splita =
542         new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
543             Bytes.toBytes("ccc"));
544       HRegionInfo splitb =
545         new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
546             Bytes.toBytes("eee"));
547       // Test that when both daughter regions are in place, that we do not
548       // remove the parent.
549       Result r = createResult(parent, splita, splitb);
550       // Add a reference under splitA directory so we don't clear out the parent.
551       Path rootdir = services.getMasterFileSystem().getRootDir();
552       Path tabledir =
553         FSUtils.getTableDir(rootdir, htd.getTableName());
554       Path storedir = HStore.getStoreHomedir(tabledir, splita,
555           htd.getColumnFamilies()[0].getName());
556       Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
557       long now = System.currentTimeMillis();
558       // Reference name has this format: StoreFile#REF_NAME_PARSER
559       Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
560       FileSystem fs = services.getMasterFileSystem().getFileSystem();
561       Path path = ref.write(fs, p);
562       assertTrue(fs.exists(path));
563       assertFalse(janitor.cleanParent(parent, r));
564       // Remove the reference file and try again.
565       assertTrue(fs.delete(p, true));
566       assertTrue(janitor.cleanParent(parent, r));
567     } finally {
568       server.stop("shutdown");
569     }
570   }
571 
572   /**
573    * Make sure parent gets cleaned up even if daughter is cleaned up before it.
574    * @throws IOException
575    * @throws InterruptedException
576    */
577   @Test
578   public void testParentCleanedEvenIfDaughterGoneFirst()
579   throws IOException, InterruptedException {
580     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
581       "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
582   }
583 
584   /**
585    * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it.
586    * @throws IOException
587    * @throws InterruptedException
588    */
589   @Test
590   public void testLastParentCleanedEvenIfDaughterGoneFirst()
591   throws IOException, InterruptedException {
592     parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
593       "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
594   }
595 
596   /**
597    * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before it.
598    *
599    * @param rootDir the test case name, used as the HBase testing utility root
600    * @param lastEndKey the end key of the split parent
601    * @throws IOException
602    * @throws InterruptedException
603    */
604   private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
605   final String rootDir, final byte[] lastEndKey)
606   throws IOException, InterruptedException {
607     HBaseTestingUtility htu = new HBaseTestingUtility();
608     setRootDirAndCleanIt(htu, rootDir);
609     Server server = new MockServer(htu);
610     MasterServices services = new MockMasterServices(server);
611     CatalogJanitor janitor = new CatalogJanitor(server, services);
612     final HTableDescriptor htd = createHTableDescriptor();
613 
614     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
615 
616     // Parent
617     HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
618       lastEndKey);
619     // Sleep a second else the encoded name on these regions comes out
620     // same for all with same start key and made in same second.
621     Thread.sleep(1001);
622 
623     // Daughter a
624     HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
625       Bytes.toBytes("ccc"));
626     Thread.sleep(1001);
627     // Make daughters of daughter a; splitaa and splitab.
628     HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
629       Bytes.toBytes("bbb"));
630     HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
631       Bytes.toBytes("ccc"));
632 
633     // Daughter b
634     HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
635       lastEndKey);
636     Thread.sleep(1001);
637     // Make Daughters of daughterb; splitba and splitbb.
638     HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
639       Bytes.toBytes("ddd"));
640     HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
641     lastEndKey);
642 
643     // First test that our Comparator works right up in CatalogJanitor.
644     // Just fo kicks.
645     SortedMap<HRegionInfo, Result> regions =
646       new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
647     // Now make sure that this regions map sorts as we expect it to.
648     regions.put(parent, createResult(parent, splita, splitb));
649     regions.put(splitb, createResult(splitb, splitba, splitbb));
650     regions.put(splita, createResult(splita, splitaa, splitab));
651     // Assert its properly sorted.
652     int index = 0;
653     for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
654       if (index == 0) {
655         assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
656       } else if (index == 1) {
657         assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
658       } else if (index == 2) {
659         assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
660       }
661       index++;
662     }
663 
664     // Now play around with the cleanParent function.  Create a ref from splita
665     // up to the parent.
666     Path splitaRef =
667       createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
668     // Make sure actual super parent sticks around because splita has a ref.
669     assertFalse(janitor.cleanParent(parent, regions.get(parent)));
670 
671     //splitba, and split bb, do not have dirs in fs.  That means that if
672     // we test splitb, it should get cleaned up.
673     assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
674 
675     // Now remove ref from splita to parent... so parent can be let go and so
676     // the daughter splita can be split (can't split if still references).
677     // BUT make the timing such that the daughter gets cleaned up before we
678     // can get a chance to let go of the parent.
679     FileSystem fs = FileSystem.get(htu.getConfiguration());
680     assertTrue(fs.delete(splitaRef, true));
681     // Create the refs from daughters of splita.
682     Path splitaaRef =
683       createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
684     Path splitabRef =
685       createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
686 
687     // Test splita.  It should stick around because references from splitab, etc.
688     assertFalse(janitor.cleanParent(splita, regions.get(splita)));
689 
690     // Now clean up parent daughter first.  Remove references from its daughters.
691     assertTrue(fs.delete(splitaaRef, true));
692     assertTrue(fs.delete(splitabRef, true));
693     assertTrue(janitor.cleanParent(splita, regions.get(splita)));
694 
695     // Super parent should get cleaned up now both splita and splitb are gone.
696     assertTrue(janitor.cleanParent(parent, regions.get(parent)));
697 
698     services.stop("test finished");
699     janitor.cancel(true);
700   }
701 
702   /**
703    * CatalogJanitor.scan() should not clean parent regions if their own
704    * parents are still referencing them. This ensures that grandfather regions
705    * do not point to deleted parent regions.
706    */
707   @Test
708   public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
709     HBaseTestingUtility htu = new HBaseTestingUtility();
710     setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
711     Server server = new MockServer(htu);
712     MasterServices services = new MockMasterServices(server);
713 
714     final HTableDescriptor htd = createHTableDescriptor();
715 
716     // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
717 
718     // Parent
719     HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
720       new byte[0], true);
721     // Sleep a second else the encoded name on these regions comes out
722     // same for all with same start key and made in same second.
723     Thread.sleep(1001);
724 
725     // Daughter a
726     HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
727       Bytes.toBytes("ccc"), true);
728     Thread.sleep(1001);
729     // Make daughters of daughter a; splitaa and splitab.
730     HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
731       Bytes.toBytes("bbb"), false);
732     HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
733       Bytes.toBytes("ccc"), false);
734 
735     // Daughter b
736     HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
737         new byte[0]);
738     Thread.sleep(1001);
739 
740     final Map<HRegionInfo, Result> splitParents =
741         new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
742     splitParents.put(parent, createResult(parent, splita, splitb));
743     splita.setOffline(true); //simulate that splita goes offline when it is split
744     splitParents.put(splita, createResult(splita, splitaa,splitab));
745 
746     final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
747     CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
748     doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
749             10, mergedRegions, splitParents)).when(janitor)
750         .getMergedRegionsAndSplitParents();
751 
752     //create ref from splita to parent
753     Path splitaRef =
754         createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
755 
756     //parent and A should not be removed
757     assertEquals(0, janitor.scan());
758 
759     //now delete the ref
760     FileSystem fs = FileSystem.get(htu.getConfiguration());
761     assertTrue(fs.delete(splitaRef, true));
762 
763     //now, both parent, and splita can be deleted
764     assertEquals(2, janitor.scan());
765 
766     services.stop("test finished");
767     janitor.cancel(true);
768   }
769 
770   /**
771    * Test that we correctly archive all the storefiles when a region is deleted
772    * @throws Exception
773    */
774   @Test
775   public void testSplitParentFirstComparator() {
776     SplitParentFirstComparator comp = new SplitParentFirstComparator();
777     final HTableDescriptor htd = createHTableDescriptor();
778 
779     /*  Region splits:
780      *
781      *  rootRegion --- firstRegion --- firstRegiona
782      *              |               |- firstRegionb
783      *              |
784      *              |- lastRegion --- lastRegiona  --- lastRegionaa
785      *                             |                |- lastRegionab
786      *                             |- lastRegionb
787      *
788      *  rootRegion   :   []  - []
789      *  firstRegion  :   []  - bbb
790      *  lastRegion   :   bbb - []
791      *  firstRegiona :   []  - aaa
792      *  firstRegionb :   aaa - bbb
793      *  lastRegiona  :   bbb - ddd
794      *  lastRegionb  :   ddd - []
795      */
796 
797     // root region
798     HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
799       HConstants.EMPTY_START_ROW,
800       HConstants.EMPTY_END_ROW, true);
801     HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
802       HConstants.EMPTY_START_ROW,
803       Bytes.toBytes("bbb"), true);
804     HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
805       Bytes.toBytes("bbb"),
806       HConstants.EMPTY_END_ROW, true);
807 
808     assertTrue(comp.compare(rootRegion, rootRegion) == 0);
809     assertTrue(comp.compare(firstRegion, firstRegion) == 0);
810     assertTrue(comp.compare(lastRegion, lastRegion) == 0);
811     assertTrue(comp.compare(rootRegion, firstRegion) < 0);
812     assertTrue(comp.compare(rootRegion, lastRegion) < 0);
813     assertTrue(comp.compare(firstRegion, lastRegion) < 0);
814 
815     //first region split into a, b
816     HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
817       HConstants.EMPTY_START_ROW,
818       Bytes.toBytes("aaa"), true);
819     HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
820         Bytes.toBytes("aaa"),
821       Bytes.toBytes("bbb"), true);
822     //last region split into a, b
823     HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
824       Bytes.toBytes("bbb"),
825       Bytes.toBytes("ddd"), true);
826     HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
827       Bytes.toBytes("ddd"),
828       HConstants.EMPTY_END_ROW, true);
829 
830     assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
831     assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
832     assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
833     assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
834     assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
835     assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
836     assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
837 
838     assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
839     assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
840     assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
841     assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
842     assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
843     assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
844     assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
845 
846     assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
847     assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
848     assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
849     assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
850 
851     HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
852       Bytes.toBytes("bbb"),
853       Bytes.toBytes("ccc"), false);
854     HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
855       Bytes.toBytes("ccc"),
856       Bytes.toBytes("ddd"), false);
857 
858     assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
859     assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
860     assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
861 
862   }
863 
864   @Test
865   public void testArchiveOldRegion() throws Exception {
866     String table = "table";
867     HBaseTestingUtility htu = new HBaseTestingUtility();
868     setRootDirAndCleanIt(htu, "testCleanParent");
869     Server server = new MockServer(htu);
870     MasterServices services = new MockMasterServices(server);
871 
872     // create the janitor
873     CatalogJanitor janitor = new CatalogJanitor(server, services);
874 
875     // Create regions.
876     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
877     htd.addFamily(new HColumnDescriptor("f"));
878     HRegionInfo parent = new HRegionInfo(htd.getTableName(),
879         Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
880     HRegionInfo splita = new HRegionInfo(htd.getTableName(),
881         Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
882     HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
883         Bytes.toBytes("ccc"),
884         Bytes.toBytes("eee"));
885 
886     // Test that when both daughter regions are in place, that we do not
887     // remove the parent.
888     Result parentMetaRow = createResult(parent, splita, splitb);
889     FileSystem fs = FileSystem.get(htu.getConfiguration());
890     Path rootdir = services.getMasterFileSystem().getRootDir();
891     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
892     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
893     // the single test passes, but when the full suite is run, things get borked).
894     FSUtils.setRootDir(fs.getConf(), rootdir);
895     Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
896     Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
897     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
898       tabledir, htd.getColumnFamilies()[0].getName());
899     LOG.debug("Table dir:" + tabledir);
900     LOG.debug("Store dir:" + storedir);
901     LOG.debug("Store archive dir:" + storeArchive);
902 
903     // add a couple of store files that we can check for
904     FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
905     // get the current store files for comparison
906     FileStatus[] storeFiles = fs.listStatus(storedir);
907     int index = 0;
908     for (FileStatus file : storeFiles) {
909       LOG.debug("Have store file:" + file.getPath());
910       assertEquals("Got unexpected store file", mockFiles[index].getPath(),
911         storeFiles[index].getPath());
912       index++;
913     }
914 
915     // do the cleaning of the parent
916     assertTrue(janitor.cleanParent(parent, parentMetaRow));
917     LOG.debug("Finished cleanup of parent region");
918 
919     // and now check to make sure that the files have actually been archived
920     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
921     logFiles("archived files", storeFiles);
922     logFiles("archived files", archivedStoreFiles);
923 
924     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
925 
926     // cleanup
927     FSUtils.delete(fs, rootdir, true);
928     services.stop("Test finished");
929     server.stop("Test finished");
930     janitor.cancel(true);
931   }
932 
933   /**
934    * @param description description of the files for logging
935    * @param storeFiles the status of the files to log
936    */
937   private void logFiles(String description, FileStatus[] storeFiles) {
938     LOG.debug("Current " + description + ": ");
939     for (FileStatus file : storeFiles) {
940       LOG.debug(file.getPath());
941     }
942   }
943 
944   /**
945    * Test that if a store file with the same name is present as those already backed up cause the
946    * already archived files to be timestamped backup
947    */
948   @Test
949   public void testDuplicateHFileResolution() throws Exception {
950     String table = "table";
951     HBaseTestingUtility htu = new HBaseTestingUtility();
952     setRootDirAndCleanIt(htu, "testCleanParent");
953     Server server = new MockServer(htu);
954     MasterServices services = new MockMasterServices(server);
955 
956     // create the janitor
957 
958     CatalogJanitor janitor = new CatalogJanitor(server, services);
959 
960     // Create regions.
961     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
962     htd.addFamily(new HColumnDescriptor("f"));
963     HRegionInfo parent = new HRegionInfo(htd.getTableName(),
964         Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
965     HRegionInfo splita = new HRegionInfo(htd.getTableName(),
966         Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
967     HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
968         Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
969     // Test that when both daughter regions are in place, that we do not
970     // remove the parent.
971     Result r = createResult(parent, splita, splitb);
972 
973     FileSystem fs = FileSystem.get(htu.getConfiguration());
974 
975     Path rootdir = services.getMasterFileSystem().getRootDir();
976     // have to set the root directory since we use it in HFileDisposer to figure out to get to the
977     // archive directory. Otherwise, it just seems to pick the first root directory it can find (so
978     // the single test passes, but when the full suite is run, things get borked).
979     FSUtils.setRootDir(fs.getConf(), rootdir);
980     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
981     Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
982     System.out.println("Old root:" + rootdir);
983     System.out.println("Old table:" + tabledir);
984     System.out.println("Old store:" + storedir);
985 
986     Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
987       tabledir, htd.getColumnFamilies()[0].getName());
988     System.out.println("Old archive:" + storeArchive);
989 
990     // enable archiving, make sure that files get archived
991     addMockStoreFiles(2, services, storedir);
992     // get the current store files for comparison
993     FileStatus[] storeFiles = fs.listStatus(storedir);
994     // do the cleaning of the parent
995     assertTrue(janitor.cleanParent(parent, r));
996 
997     // and now check to make sure that the files have actually been archived
998     FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
999     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
1000 
1001     // now add store files with the same names as before to check backup
1002     // enable archiving, make sure that files get archived
1003     addMockStoreFiles(2, services, storedir);
1004 
1005     // do the cleaning of the parent
1006     assertTrue(janitor.cleanParent(parent, r));
1007 
1008     // and now check to make sure that the files have actually been archived
1009     archivedStoreFiles = fs.listStatus(storeArchive);
1010     assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
1011 
1012     // cleanup
1013     services.stop("Test finished");
1014     server.stop("shutdown");
1015     janitor.cancel(true);
1016   }
1017 
1018   private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
1019       throws IOException {
1020     // get the existing store files
1021     FileSystem fs = services.getMasterFileSystem().getFileSystem();
1022     fs.mkdirs(storedir);
1023     // create the store files in the parent
1024     for (int i = 0; i < count; i++) {
1025       Path storeFile = new Path(storedir, "_store" + i);
1026       FSDataOutputStream dos = fs.create(storeFile, true);
1027       dos.writeBytes("Some data: " + i);
1028       dos.close();
1029     }
1030     LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
1031     // make sure the mock store files are there
1032     FileStatus[] storeFiles = fs.listStatus(storedir);
1033     assertEquals("Didn't have expected store files", count, storeFiles.length);
1034     return storeFiles;
1035   }
1036 
1037   private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
1038       final String subdir)
1039   throws IOException {
1040     Path testdir = htu.getDataTestDir(subdir);
1041     FileSystem fs = FileSystem.get(htu.getConfiguration());
1042     if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
1043     FSUtils.setRootDir(htu.getConfiguration(), testdir);
1044     return FSUtils.getRootDir(htu.getConfiguration()).toString();
1045   }
1046 
1047   /**
1048    * @param services Master services instance.
1049    * @param htd
1050    * @param parent
1051    * @param daughter
1052    * @param midkey
1053    * @param top True if we are to write a 'top' reference.
1054    * @return Path to reference we created.
1055    * @throws IOException
1056    */
1057   private Path createReferences(final MasterServices services,
1058       final HTableDescriptor htd, final HRegionInfo parent,
1059       final HRegionInfo daughter, final byte [] midkey, final boolean top)
1060   throws IOException {
1061     Path rootdir = services.getMasterFileSystem().getRootDir();
1062     Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1063     Path storedir = HStore.getStoreHomedir(tabledir, daughter,
1064       htd.getColumnFamilies()[0].getName());
1065     Reference ref =
1066       top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
1067     long now = System.currentTimeMillis();
1068     // Reference name has this format: StoreFile#REF_NAME_PARSER
1069     Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
1070     FileSystem fs = services.getMasterFileSystem().getFileSystem();
1071     ref.write(fs, p);
1072     return p;
1073   }
1074 
1075   private Result createResult(final HRegionInfo parent, final HRegionInfo a,
1076       final HRegionInfo b)
1077   throws IOException {
1078     return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
1079   }
1080 
1081   private HTableDescriptor createHTableDescriptor() {
1082     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
1083     htd.addFamily(new HColumnDescriptor("f"));
1084     return htd;
1085   }
1086 
1087   private MultiResponse buildMultiResponse(MultiRequest req) {
1088     MultiResponse.Builder builder = MultiResponse.newBuilder();
1089     RegionActionResult.Builder regionActionResultBuilder =
1090         RegionActionResult.newBuilder();
1091     ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
1092     for (RegionAction regionAction: req.getRegionActionList()) {
1093       regionActionResultBuilder.clear();
1094       for (ClientProtos.Action action: regionAction.getActionList()) {
1095         roeBuilder.clear();
1096         roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
1097         roeBuilder.setIndex(action.getIndex());
1098         regionActionResultBuilder.addResultOrException(roeBuilder.build());
1099       }
1100       builder.addRegionActionResult(regionActionResultBuilder.build());
1101     }
1102     return builder.build();
1103   }
1104 
1105 }
1106