1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.spy;
27
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.SortedMap;
32 import java.util.TreeMap;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.ChoreService;
42 import org.apache.hadoop.hbase.CoordinatedStateManager;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.MetaMockingUtil;
49 import org.apache.hadoop.hbase.NamespaceDescriptor;
50 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
51 import org.apache.hadoop.hbase.ProcedureInfo;
52 import org.apache.hadoop.hbase.Server;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.testclassification.SmallTests;
55 import org.apache.hadoop.hbase.TableDescriptors;
56 import org.apache.hadoop.hbase.TableName;
57 import org.apache.hadoop.hbase.TableStateManager;
58 import org.apache.hadoop.hbase.backup.BackupType;
59 import org.apache.hadoop.hbase.client.ClusterConnection;
60 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
61 import org.apache.hadoop.hbase.client.Result;
62 import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
63 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
64 import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
65 import org.apache.hadoop.hbase.executor.ExecutorService;
66 import org.apache.hadoop.hbase.io.Reference;
67 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
68 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
69 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
70 import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
71 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
72 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
73 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
74 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
75 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
76 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
77 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
78 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
79 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
80 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
81 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
82 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
83 import org.apache.hadoop.hbase.regionserver.HStore;
84 import org.apache.hadoop.hbase.util.Bytes;
85 import org.apache.hadoop.hbase.util.FSUtils;
86 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
87 import org.apache.hadoop.hbase.util.Pair;
88 import org.apache.hadoop.hbase.util.Triple;
89 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
90 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
91 import org.junit.Test;
92 import org.junit.experimental.categories.Category;
93 import org.mockito.Mockito;
94 import org.mockito.invocation.InvocationOnMock;
95 import org.mockito.stubbing.Answer;
96
97 import com.google.protobuf.RpcController;
98 import com.google.protobuf.Service;
99 import com.google.protobuf.ServiceException;
100
101 @Category(SmallTests.class)
102 public class TestCatalogJanitor {
103 private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
104
105
106
107
108
109 class MockServer implements Server {
110 private final ClusterConnection connection;
111 private final Configuration c;
112
113 MockServer(final HBaseTestingUtility htu)
114 throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
115 this.c = htu.getConfiguration();
116 ClientProtos.ClientService.BlockingInterface ri =
117 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
118 MutateResponse.Builder builder = MutateResponse.newBuilder();
119 builder.setProcessed(true);
120 try {
121 Mockito.when(ri.mutate(
122 (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
123 thenReturn(builder.build());
124 } catch (ServiceException se) {
125 throw ProtobufUtil.getRemoteException(se);
126 }
127 try {
128 Mockito.when(ri.multi(
129 (RpcController)Mockito.any(), (MultiRequest)Mockito.any())).
130 thenAnswer(new Answer<MultiResponse>() {
131 @Override
132 public MultiResponse answer(InvocationOnMock invocation) throws Throwable {
133 return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]);
134 }
135 });
136 } catch (ServiceException se) {
137 throw ProtobufUtil.getRemoteException(se);
138 }
139
140
141
142 this.connection =
143 HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
144 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
145 ServerName.valueOf("example.org,12345,6789"),
146 HRegionInfo.FIRST_META_REGIONINFO);
147
148 FileSystem fs = FileSystem.get(this.c);
149 Path rootdir = FSUtils.getRootDir(this.c);
150 FSUtils.setRootDir(this.c, rootdir);
151 AdminProtos.AdminService.BlockingInterface hri =
152 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
153 }
154
155 @Override
156 public ClusterConnection getConnection() {
157 return this.connection;
158 }
159
160 @Override
161 public MetaTableLocator getMetaTableLocator() {
162 return null;
163 }
164
165 @Override
166 public Configuration getConfiguration() {
167 return this.c;
168 }
169
170 @Override
171 public ServerName getServerName() {
172 return ServerName.valueOf("mockserver.example.org", 1234, -1L);
173 }
174
175 @Override
176 public ZooKeeperWatcher getZooKeeper() {
177 return null;
178 }
179
180 @Override
181 public CoordinatedStateManager getCoordinatedStateManager() {
182 BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class);
183 SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class);
184 Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c);
185 SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class);
186 Mockito.when(c.getDetails()).thenReturn(d);
187 return m;
188 }
189
190 @Override
191 public void abort(String why, Throwable e) {
192
193 }
194
195 @Override
196 public boolean isAborted() {
197 return false;
198 }
199
200 @Override
201 public boolean isStopped() {
202 return false;
203 }
204
205 @Override
206 public void stop(String why) {
207 }
208
209 @Override
210 public ChoreService getChoreService() {
211 return null;
212 }
213 }
214
215
216
217
218 class MockMasterServices implements MasterServices {
219 private final MasterFileSystem mfs;
220 private final AssignmentManager asm;
221
222 MockMasterServices(final Server server) throws IOException {
223 this.mfs = new MasterFileSystem(server, this);
224 this.asm = Mockito.mock(AssignmentManager.class);
225 }
226
227 @Override
228 public void checkTableModifiable(TableName tableName) throws IOException {
229
230 }
231
232 @Override
233 public boolean isMasterProcedureExecutorEnabled() {
234 return true;
235 }
236
237 @Override
238 public long createTable(HTableDescriptor desc, byte[][] splitKeys)
239 throws IOException {
240
241 return -1;
242 }
243
244 @Override
245 public SnapshotManager getSnapshotManager() {
246 return null;
247 }
248
249 @Override
250 public MasterProcedureManagerHost getMasterProcedureManagerHost() {
251 return null;
252 }
253
254 @Override
255 public AssignmentManager getAssignmentManager() {
256 return this.asm;
257 }
258
259 @Override
260 public ExecutorService getExecutorService() {
261 return null;
262 }
263
264 @Override
265 public ChoreService getChoreService() {
266 return null;
267 }
268
269 @Override
270 public MasterFileSystem getMasterFileSystem() {
271 return this.mfs;
272 }
273
274 @Override
275 public MasterCoprocessorHost getMasterCoprocessorHost() {
276 return null;
277 }
278
279 @Override
280 public MasterQuotaManager getMasterQuotaManager() {
281 return null;
282 }
283
284 @Override
285 public ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
286 return null;
287 }
288
289 @Override
290 public ServerManager getServerManager() {
291 return null;
292 }
293
294 @Override
295 public ZooKeeperWatcher getZooKeeper() {
296 return null;
297 }
298
299 @Override
300 public CoordinatedStateManager getCoordinatedStateManager() {
301 return null;
302 }
303
304 @Override
305 public MetaTableLocator getMetaTableLocator() {
306 return null;
307 }
308
309 @Override
310 public ClusterConnection getConnection() {
311 return null;
312 }
313
314 @Override
315 public Configuration getConfiguration() {
316 return mfs.conf;
317 }
318
319 @Override
320 public ServerName getServerName() {
321 return null;
322 }
323
324 @Override
325 public void abort(String why, Throwable e) {
326
327 }
328
329 @Override
330 public boolean isAborted() {
331 return false;
332 }
333
334 private boolean stopped = false;
335
336 @Override
337 public void stop(String why) {
338 stopped = true;
339 }
340
341 @Override
342 public boolean isStopped() {
343 return stopped;
344 }
345
346 @Override
347 public TableDescriptors getTableDescriptors() {
348 return new TableDescriptors() {
349 @Override
350 public HTableDescriptor remove(TableName tablename) throws IOException {
351
352 return null;
353 }
354
355 @Override
356 public Map<String, HTableDescriptor> getAll() throws IOException {
357
358 return null;
359 }
360
361 @Override
362 public HTableDescriptor get(TableName tablename)
363 throws IOException {
364 return createHTableDescriptor();
365 }
366
367 @Override
368 public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
369 return null;
370 }
371
372 @Override
373 public void add(HTableDescriptor htd) throws IOException {
374
375
376 }
377 @Override
378 public void setCacheOn() throws IOException {
379 }
380
381 @Override
382 public void setCacheOff() throws IOException {
383 }
384 };
385 }
386
387 @Override
388 public boolean isServerShutdownHandlerEnabled() {
389 return true;
390 }
391
392 @Override
393 public boolean registerService(Service instance) {
394 return false;
395 }
396
397 @Override
398 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
399
400 }
401
402 @Override
403 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
404
405 }
406
407 @Override
408 public void deleteNamespace(String name) throws IOException {
409
410 }
411
412 @Override
413 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
414 return null;
415 }
416
417 @Override
418 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
419 return null;
420 }
421
422 @Override
423 public boolean abortProcedure(final long procId, final boolean mayInterruptIfRunning)
424 throws IOException {
425 return false;
426 }
427
428 @Override
429 public List<ProcedureInfo> listProcedures() throws IOException {
430 return null;
431 }
432
433 @Override
434 public Pair<Long, String> backupTables(
435 final BackupType type,
436 final List<TableName> tableList,
437 final String targetRootDir, final int workers,
438 final long bandwidth) throws IOException {
439 return null;
440 }
441
442
443 @Override
444 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
445 return null;
446 }
447
448 @Override
449 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
450 return null;
451 }
452
453 @Override
454 public long deleteTable(TableName tableName) throws IOException {
455 return -1;
456 }
457 @Override
458 public LoadBalancer getLoadBalancer() {
459 return null;
460 }
461
462 @Override
463 public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException { }
464
465
466 @Override
467 public void modifyTable(TableName tableName, HTableDescriptor descriptor)
468 throws IOException { }
469
470 @Override
471 public long enableTable(TableName tableName) throws IOException {
472 return -1;
473 }
474
475 @Override
476 public long disableTable(TableName tableName) throws IOException {
477 return -1;
478 }
479
480 @Override
481 public void addColumn(TableName tableName, HColumnDescriptor column)
482 throws IOException { }
483
484 @Override
485 public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
486 throws IOException { }
487
488 @Override
489 public void deleteColumn(TableName tableName, byte[] columnName)
490 throws IOException { }
491
492 @Override
493 public TableLockManager getTableLockManager() {
494 return null;
495 }
496
497 @Override
498 public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
499 boolean forcible) throws IOException {
500 }
501
502 @Override
503 public boolean isInitialized() {
504
505 return false;
506 }
507
508 @Override
509 public long getLastMajorCompactionTimestamp(TableName table) throws IOException {
510
511 return 0;
512 }
513
514 @Override
515 public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
516
517 return 0;
518 }
519
520 @Override
521 public TableStateManager getTableStateManager() {
522
523 return null;
524 }
525 }
526
527 @Test
528 public void testCleanParent() throws IOException, InterruptedException {
529 HBaseTestingUtility htu = new HBaseTestingUtility();
530 setRootDirAndCleanIt(htu, "testCleanParent");
531 Server server = new MockServer(htu);
532 try {
533 MasterServices services = new MockMasterServices(server);
534 CatalogJanitor janitor = new CatalogJanitor(server, services);
535
536 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
537 htd.addFamily(new HColumnDescriptor("f"));
538 HRegionInfo parent =
539 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
540 Bytes.toBytes("eee"));
541 HRegionInfo splita =
542 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
543 Bytes.toBytes("ccc"));
544 HRegionInfo splitb =
545 new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
546 Bytes.toBytes("eee"));
547
548
549 Result r = createResult(parent, splita, splitb);
550
551 Path rootdir = services.getMasterFileSystem().getRootDir();
552 Path tabledir =
553 FSUtils.getTableDir(rootdir, htd.getTableName());
554 Path storedir = HStore.getStoreHomedir(tabledir, splita,
555 htd.getColumnFamilies()[0].getName());
556 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
557 long now = System.currentTimeMillis();
558
559 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
560 FileSystem fs = services.getMasterFileSystem().getFileSystem();
561 Path path = ref.write(fs, p);
562 assertTrue(fs.exists(path));
563 assertFalse(janitor.cleanParent(parent, r));
564
565 assertTrue(fs.delete(p, true));
566 assertTrue(janitor.cleanParent(parent, r));
567 } finally {
568 server.stop("shutdown");
569 }
570 }
571
572
573
574
575
576
577 @Test
578 public void testParentCleanedEvenIfDaughterGoneFirst()
579 throws IOException, InterruptedException {
580 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
581 "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
582 }
583
584
585
586
587
588
589 @Test
590 public void testLastParentCleanedEvenIfDaughterGoneFirst()
591 throws IOException, InterruptedException {
592 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
593 "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
594 }
595
596
597
598
599
600
601
602
603
604 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
605 final String rootDir, final byte[] lastEndKey)
606 throws IOException, InterruptedException {
607 HBaseTestingUtility htu = new HBaseTestingUtility();
608 setRootDirAndCleanIt(htu, rootDir);
609 Server server = new MockServer(htu);
610 MasterServices services = new MockMasterServices(server);
611 CatalogJanitor janitor = new CatalogJanitor(server, services);
612 final HTableDescriptor htd = createHTableDescriptor();
613
614
615
616
617 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
618 lastEndKey);
619
620
621 Thread.sleep(1001);
622
623
624 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
625 Bytes.toBytes("ccc"));
626 Thread.sleep(1001);
627
628 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
629 Bytes.toBytes("bbb"));
630 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
631 Bytes.toBytes("ccc"));
632
633
634 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
635 lastEndKey);
636 Thread.sleep(1001);
637
638 HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
639 Bytes.toBytes("ddd"));
640 HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
641 lastEndKey);
642
643
644
645 SortedMap<HRegionInfo, Result> regions =
646 new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
647
648 regions.put(parent, createResult(parent, splita, splitb));
649 regions.put(splitb, createResult(splitb, splitba, splitbb));
650 regions.put(splita, createResult(splita, splitaa, splitab));
651
652 int index = 0;
653 for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
654 if (index == 0) {
655 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
656 } else if (index == 1) {
657 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
658 } else if (index == 2) {
659 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
660 }
661 index++;
662 }
663
664
665
666 Path splitaRef =
667 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
668
669 assertFalse(janitor.cleanParent(parent, regions.get(parent)));
670
671
672
673 assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
674
675
676
677
678
679 FileSystem fs = FileSystem.get(htu.getConfiguration());
680 assertTrue(fs.delete(splitaRef, true));
681
682 Path splitaaRef =
683 createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
684 Path splitabRef =
685 createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
686
687
688 assertFalse(janitor.cleanParent(splita, regions.get(splita)));
689
690
691 assertTrue(fs.delete(splitaaRef, true));
692 assertTrue(fs.delete(splitabRef, true));
693 assertTrue(janitor.cleanParent(splita, regions.get(splita)));
694
695
696 assertTrue(janitor.cleanParent(parent, regions.get(parent)));
697
698 services.stop("test finished");
699 janitor.cancel(true);
700 }
701
702
703
704
705
706
707 @Test
708 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
709 HBaseTestingUtility htu = new HBaseTestingUtility();
710 setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
711 Server server = new MockServer(htu);
712 MasterServices services = new MockMasterServices(server);
713
714 final HTableDescriptor htd = createHTableDescriptor();
715
716
717
718
719 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
720 new byte[0], true);
721
722
723 Thread.sleep(1001);
724
725
726 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
727 Bytes.toBytes("ccc"), true);
728 Thread.sleep(1001);
729
730 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
731 Bytes.toBytes("bbb"), false);
732 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
733 Bytes.toBytes("ccc"), false);
734
735
736 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
737 new byte[0]);
738 Thread.sleep(1001);
739
740 final Map<HRegionInfo, Result> splitParents =
741 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
742 splitParents.put(parent, createResult(parent, splita, splitb));
743 splita.setOffline(true);
744 splitParents.put(splita, createResult(splita, splitaa,splitab));
745
746 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
747 CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
748 doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
749 10, mergedRegions, splitParents)).when(janitor)
750 .getMergedRegionsAndSplitParents();
751
752
753 Path splitaRef =
754 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
755
756
757 assertEquals(0, janitor.scan());
758
759
760 FileSystem fs = FileSystem.get(htu.getConfiguration());
761 assertTrue(fs.delete(splitaRef, true));
762
763
764 assertEquals(2, janitor.scan());
765
766 services.stop("test finished");
767 janitor.cancel(true);
768 }
769
770
771
772
773
774 @Test
775 public void testSplitParentFirstComparator() {
776 SplitParentFirstComparator comp = new SplitParentFirstComparator();
777 final HTableDescriptor htd = createHTableDescriptor();
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798 HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
799 HConstants.EMPTY_START_ROW,
800 HConstants.EMPTY_END_ROW, true);
801 HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
802 HConstants.EMPTY_START_ROW,
803 Bytes.toBytes("bbb"), true);
804 HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
805 Bytes.toBytes("bbb"),
806 HConstants.EMPTY_END_ROW, true);
807
808 assertTrue(comp.compare(rootRegion, rootRegion) == 0);
809 assertTrue(comp.compare(firstRegion, firstRegion) == 0);
810 assertTrue(comp.compare(lastRegion, lastRegion) == 0);
811 assertTrue(comp.compare(rootRegion, firstRegion) < 0);
812 assertTrue(comp.compare(rootRegion, lastRegion) < 0);
813 assertTrue(comp.compare(firstRegion, lastRegion) < 0);
814
815
816 HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
817 HConstants.EMPTY_START_ROW,
818 Bytes.toBytes("aaa"), true);
819 HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
820 Bytes.toBytes("aaa"),
821 Bytes.toBytes("bbb"), true);
822
823 HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
824 Bytes.toBytes("bbb"),
825 Bytes.toBytes("ddd"), true);
826 HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
827 Bytes.toBytes("ddd"),
828 HConstants.EMPTY_END_ROW, true);
829
830 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
831 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
832 assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
833 assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
834 assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
835 assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
836 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
837
838 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
839 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
840 assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
841 assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
842 assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
843 assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
844 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
845
846 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
847 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
848 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
849 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
850
851 HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
852 Bytes.toBytes("bbb"),
853 Bytes.toBytes("ccc"), false);
854 HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
855 Bytes.toBytes("ccc"),
856 Bytes.toBytes("ddd"), false);
857
858 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
859 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
860 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
861
862 }
863
864 @Test
865 public void testArchiveOldRegion() throws Exception {
866 String table = "table";
867 HBaseTestingUtility htu = new HBaseTestingUtility();
868 setRootDirAndCleanIt(htu, "testCleanParent");
869 Server server = new MockServer(htu);
870 MasterServices services = new MockMasterServices(server);
871
872
873 CatalogJanitor janitor = new CatalogJanitor(server, services);
874
875
876 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
877 htd.addFamily(new HColumnDescriptor("f"));
878 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
879 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
880 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
881 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
882 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
883 Bytes.toBytes("ccc"),
884 Bytes.toBytes("eee"));
885
886
887
888 Result parentMetaRow = createResult(parent, splita, splitb);
889 FileSystem fs = FileSystem.get(htu.getConfiguration());
890 Path rootdir = services.getMasterFileSystem().getRootDir();
891
892
893
894 FSUtils.setRootDir(fs.getConf(), rootdir);
895 Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
896 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
897 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
898 tabledir, htd.getColumnFamilies()[0].getName());
899 LOG.debug("Table dir:" + tabledir);
900 LOG.debug("Store dir:" + storedir);
901 LOG.debug("Store archive dir:" + storeArchive);
902
903
904 FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
905
906 FileStatus[] storeFiles = fs.listStatus(storedir);
907 int index = 0;
908 for (FileStatus file : storeFiles) {
909 LOG.debug("Have store file:" + file.getPath());
910 assertEquals("Got unexpected store file", mockFiles[index].getPath(),
911 storeFiles[index].getPath());
912 index++;
913 }
914
915
916 assertTrue(janitor.cleanParent(parent, parentMetaRow));
917 LOG.debug("Finished cleanup of parent region");
918
919
920 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
921 logFiles("archived files", storeFiles);
922 logFiles("archived files", archivedStoreFiles);
923
924 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
925
926
927 FSUtils.delete(fs, rootdir, true);
928 services.stop("Test finished");
929 server.stop("Test finished");
930 janitor.cancel(true);
931 }
932
933
934
935
936
937 private void logFiles(String description, FileStatus[] storeFiles) {
938 LOG.debug("Current " + description + ": ");
939 for (FileStatus file : storeFiles) {
940 LOG.debug(file.getPath());
941 }
942 }
943
944
945
946
947
948 @Test
949 public void testDuplicateHFileResolution() throws Exception {
950 String table = "table";
951 HBaseTestingUtility htu = new HBaseTestingUtility();
952 setRootDirAndCleanIt(htu, "testCleanParent");
953 Server server = new MockServer(htu);
954 MasterServices services = new MockMasterServices(server);
955
956
957
958 CatalogJanitor janitor = new CatalogJanitor(server, services);
959
960
961 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
962 htd.addFamily(new HColumnDescriptor("f"));
963 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
964 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
965 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
966 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
967 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
968 Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
969
970
971 Result r = createResult(parent, splita, splitb);
972
973 FileSystem fs = FileSystem.get(htu.getConfiguration());
974
975 Path rootdir = services.getMasterFileSystem().getRootDir();
976
977
978
979 FSUtils.setRootDir(fs.getConf(), rootdir);
980 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
981 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
982 System.out.println("Old root:" + rootdir);
983 System.out.println("Old table:" + tabledir);
984 System.out.println("Old store:" + storedir);
985
986 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
987 tabledir, htd.getColumnFamilies()[0].getName());
988 System.out.println("Old archive:" + storeArchive);
989
990
991 addMockStoreFiles(2, services, storedir);
992
993 FileStatus[] storeFiles = fs.listStatus(storedir);
994
995 assertTrue(janitor.cleanParent(parent, r));
996
997
998 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
999 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
1000
1001
1002
1003 addMockStoreFiles(2, services, storedir);
1004
1005
1006 assertTrue(janitor.cleanParent(parent, r));
1007
1008
1009 archivedStoreFiles = fs.listStatus(storeArchive);
1010 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
1011
1012
1013 services.stop("Test finished");
1014 server.stop("shutdown");
1015 janitor.cancel(true);
1016 }
1017
1018 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
1019 throws IOException {
1020
1021 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1022 fs.mkdirs(storedir);
1023
1024 for (int i = 0; i < count; i++) {
1025 Path storeFile = new Path(storedir, "_store" + i);
1026 FSDataOutputStream dos = fs.create(storeFile, true);
1027 dos.writeBytes("Some data: " + i);
1028 dos.close();
1029 }
1030 LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
1031
1032 FileStatus[] storeFiles = fs.listStatus(storedir);
1033 assertEquals("Didn't have expected store files", count, storeFiles.length);
1034 return storeFiles;
1035 }
1036
1037 private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
1038 final String subdir)
1039 throws IOException {
1040 Path testdir = htu.getDataTestDir(subdir);
1041 FileSystem fs = FileSystem.get(htu.getConfiguration());
1042 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
1043 FSUtils.setRootDir(htu.getConfiguration(), testdir);
1044 return FSUtils.getRootDir(htu.getConfiguration()).toString();
1045 }
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057 private Path createReferences(final MasterServices services,
1058 final HTableDescriptor htd, final HRegionInfo parent,
1059 final HRegionInfo daughter, final byte [] midkey, final boolean top)
1060 throws IOException {
1061 Path rootdir = services.getMasterFileSystem().getRootDir();
1062 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
1063 Path storedir = HStore.getStoreHomedir(tabledir, daughter,
1064 htd.getColumnFamilies()[0].getName());
1065 Reference ref =
1066 top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
1067 long now = System.currentTimeMillis();
1068
1069 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
1070 FileSystem fs = services.getMasterFileSystem().getFileSystem();
1071 ref.write(fs, p);
1072 return p;
1073 }
1074
1075 private Result createResult(final HRegionInfo parent, final HRegionInfo a,
1076 final HRegionInfo b)
1077 throws IOException {
1078 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
1079 }
1080
1081 private HTableDescriptor createHTableDescriptor() {
1082 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
1083 htd.addFamily(new HColumnDescriptor("f"));
1084 return htd;
1085 }
1086
1087 private MultiResponse buildMultiResponse(MultiRequest req) {
1088 MultiResponse.Builder builder = MultiResponse.newBuilder();
1089 RegionActionResult.Builder regionActionResultBuilder =
1090 RegionActionResult.newBuilder();
1091 ResultOrException.Builder roeBuilder = ResultOrException.newBuilder();
1092 for (RegionAction regionAction: req.getRegionActionList()) {
1093 regionActionResultBuilder.clear();
1094 for (ClientProtos.Action action: regionAction.getActionList()) {
1095 roeBuilder.clear();
1096 roeBuilder.setResult(ClientProtos.Result.getDefaultInstance());
1097 roeBuilder.setIndex(action.getIndex());
1098 regionActionResultBuilder.addResultOrException(roeBuilder.build());
1099 }
1100 builder.addRegionActionResult(regionActionResultBuilder.build());
1101 }
1102 return builder.build();
1103 }
1104
1105 }
1106