Skip to content

Commit 4cad08d

Browse files
gvprathyusha6gvprathyusha6
authored
HBASE-28564 Refactor direct interactions of Reference file creations to SFT interface (#5939) (#7382)
Signed-off-by: Andrew Purtell <apurtell@apache.org> Conflicts: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/janitor/TestCatalogJanitor.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobStoreCompaction.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionFileSystem.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
1 parent 26cbc3e commit 4cad08d

75 files changed

Lines changed: 1133 additions & 466 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,7 @@ public static String createFromHFileLink(final Configuration conf, final FileSys
463463
* Create the back reference name
464464
*/
465465
// package-private for testing
466-
static String createBackReferenceName(final String tableNameStr, final String regionName) {
466+
public static String createBackReferenceName(final String tableNameStr, final String regionName) {
467467

468468
return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '=');
469469
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ public static Reference convert(final FSProtos.Reference r) {
195195
* delimiter, pb reads to EOF which may not be what you want).
196196
* @return This instance serialized as a delimited protobuf w/ a magic pb prefix.
197197
*/
198-
byte[] toByteArray() throws IOException {
198+
public byte[] toByteArray() throws IOException {
199199
return ProtobufUtil.prependPBMagic(convert().toByteArray());
200200
}
201201

‎hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,6 @@ public class CacheConfig implements PropagatingConfigurationObserver {
162162

163163
private final ByteBuffAllocator byteBuffAllocator;
164164

165-
166165
/**
167166
* Create a cache configuration using the specified configuration object and defaults for family
168167
* level settings. Only use if no column family context.

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -644,7 +644,7 @@ private List<Path> mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg
644644
// to read the hfiles.
645645
storeFileInfo.setConf(storeConfiguration);
646646
Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
647-
new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
647+
new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker);
648648
mergedFiles.add(refFile);
649649
}
650650
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -701,8 +701,9 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
701701
// table dir. In case of failure, the proc would go through this again, already existing
702702
// region dirs and split files would just be ignored, new split files should get created.
703703
int nbFiles = 0;
704-
final Map<String, Collection<StoreFileInfo>> files =
705-
new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
704+
final Map<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>> files =
705+
new HashMap<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>>(
706+
htd.getColumnFamilyCount());
706707
for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
707708
String family = cfd.getNameAsString();
708709
StoreFileTracker tracker =
@@ -725,7 +726,7 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
725726
}
726727
if (filteredSfis == null) {
727728
filteredSfis = new ArrayList<StoreFileInfo>(sfis.size());
728-
files.put(family, filteredSfis);
729+
files.put(family, new Pair(filteredSfis, tracker));
729730
}
730731
filteredSfis.add(sfi);
731732
nbFiles++;
@@ -748,10 +749,12 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
748749
final List<Future<Pair<Path, Path>>> futures = new ArrayList<Future<Pair<Path, Path>>>(nbFiles);
749750

750751
// Split each store file.
751-
for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
752+
for (Map.Entry<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>> e : files
753+
.entrySet()) {
752754
byte[] familyName = Bytes.toBytes(e.getKey());
753755
final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName);
754-
final Collection<StoreFileInfo> storeFiles = e.getValue();
756+
Pair<Collection<StoreFileInfo>, StoreFileTracker> storeFilesAndTracker = e.getValue();
757+
final Collection<StoreFileInfo> storeFiles = storeFilesAndTracker.getFirst();
755758
if (storeFiles != null && storeFiles.size() > 0) {
756759
final Configuration storeConfiguration =
757760
StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
@@ -762,8 +765,9 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
762765
// is running in a regionserver's Store context, or we might not be able
763766
// to read the hfiles.
764767
storeFileInfo.setConf(storeConfiguration);
765-
StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName,
766-
new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
768+
StoreFileSplitter sfs =
769+
new StoreFileSplitter(regionFs, storeFilesAndTracker.getSecond(), familyName,
770+
new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
767771
futures.add(threadPool.submit(sfs));
768772
}
769773
}
@@ -829,19 +833,19 @@ private void assertSplitResultFilesCount(final FileSystem fs,
829833
}
830834
}
831835

832-
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
833-
throws IOException {
836+
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, StoreFileTracker tracker,
837+
byte[] family, HStoreFile sf) throws IOException {
834838
if (LOG.isDebugEnabled()) {
835839
LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath()
836840
+ " for region: " + getParentRegion().getShortNameToLog());
837841
}
838842

839843
final byte[] splitRow = getSplitRow();
840844
final String familyName = Bytes.toString(family);
841-
final Path path_first =
842-
regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, false, splitPolicy);
843-
final Path path_second =
844-
regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, true, splitPolicy);
845+
final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
846+
false, splitPolicy, tracker);
847+
final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
848+
true, splitPolicy, tracker);
845849
if (LOG.isDebugEnabled()) {
846850
LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath()
847851
+ " for region: " + getParentRegion().getShortNameToLog());
@@ -857,22 +861,25 @@ private class StoreFileSplitter implements Callable<Pair<Path, Path>> {
857861
private final HRegionFileSystem regionFs;
858862
private final byte[] family;
859863
private final HStoreFile sf;
864+
private final StoreFileTracker tracker;
860865

861866
/**
862867
* Constructor that takes what it needs to split
863868
* @param regionFs the file system
864869
* @param family Family that contains the store file
865870
* @param sf which file
866871
*/
867-
public StoreFileSplitter(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) {
872+
public StoreFileSplitter(HRegionFileSystem regionFs, StoreFileTracker tracker, byte[] family,
873+
HStoreFile sf) {
868874
this.regionFs = regionFs;
869875
this.sf = sf;
870876
this.family = family;
877+
this.tracker = tracker;
871878
}
872879

873880
@Override
874881
public Pair<Path, Path> call() throws IOException {
875-
return splitStoreFile(regionFs, family, sf);
882+
return splitStoreFile(regionFs, tracker, family, sf);
876883
}
877884
}
878885

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import org.apache.hadoop.hbase.MetaTableAccessor;
3535
import org.apache.hadoop.hbase.ScheduledChore;
3636
import org.apache.hadoop.hbase.TableName;
37+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
38+
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
3739
import org.apache.hadoop.hbase.client.Connection;
3840
import org.apache.hadoop.hbase.client.ConnectionFactory;
3941
import org.apache.hadoop.hbase.client.Get;
@@ -49,6 +51,8 @@
4951
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
5052
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
5153
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
54+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
55+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
5256
import org.apache.hadoop.hbase.util.Bytes;
5357
import org.apache.hadoop.hbase.util.CommonFSUtils;
5458
import org.apache.hadoop.hbase.util.Pair;
@@ -421,7 +425,16 @@ private static Pair<Boolean, Boolean> checkRegionReferences(MasterServices servi
421425
try {
422426
HRegionFileSystem regionFs = HRegionFileSystem
423427
.openRegionFromFileSystem(services.getConfiguration(), fs, tabledir, region, true);
424-
boolean references = regionFs.hasReferences(tableDescriptor);
428+
ColumnFamilyDescriptor[] families = tableDescriptor.getColumnFamilies();
429+
boolean references = false;
430+
for (ColumnFamilyDescriptor cfd : families) {
431+
StoreFileTracker sft = StoreFileTrackerFactory.create(services.getConfiguration(),
432+
tableDescriptor, ColumnFamilyDescriptorBuilder.of(cfd.getNameAsString()), regionFs);
433+
references = references || sft.hasReferences();
434+
if (references) {
435+
break;
436+
}
437+
}
425438
return new Pair<>(Boolean.TRUE, references);
426439
} catch (IOException e) {
427440
LOG.error("Error trying to determine if region {} has references, assuming it does",

‎hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
2626
import org.apache.hadoop.hbase.regionserver.BloomType;
2727
import org.apache.hadoop.hbase.regionserver.HStoreFile;
28+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
2829
import org.apache.yetus.audience.InterfaceAudience;
2930

3031
/**
@@ -41,10 +42,10 @@ public CachedMobFile(HStoreFile sf) {
4142
}
4243

4344
public static CachedMobFile create(FileSystem fs, Path path, Configuration conf,
44-
CacheConfig cacheConf) throws IOException {
45+
CacheConfig cacheConf, StoreFileTracker sft) throws IOException {
4546
// XXX: primaryReplica is only used for constructing the key of block cache so it is not a
4647
// critical problem if we pass the wrong value, so here we always pass true. Need to fix later.
47-
HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
48+
HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft);
4849
return new CachedMobFile(sf);
4950
}
5051

‎hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,17 +57,17 @@ public class ExpiredMobFileCleaner extends Configured implements Tool {
5757
* @param tableName The current table name.
5858
* @param family The current family.
5959
*/
60-
public void cleanExpiredMobFiles(String tableName, ColumnFamilyDescriptor family)
60+
public void cleanExpiredMobFiles(TableDescriptor htd, ColumnFamilyDescriptor family)
6161
throws IOException {
6262
Configuration conf = getConf();
63-
TableName tn = TableName.valueOf(tableName);
63+
String tableName = htd.getTableName().getNameAsString();
6464
FileSystem fs = FileSystem.get(conf);
6565
LOG.info("Cleaning the expired MOB files of " + family.getNameAsString() + " in " + tableName);
6666
// disable the block cache.
6767
Configuration copyOfConf = new Configuration(conf);
6868
copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
6969
CacheConfig cacheConfig = new CacheConfig(copyOfConf);
70-
MobUtils.cleanExpiredMobFiles(fs, conf, tn, family, cacheConfig,
70+
MobUtils.cleanExpiredMobFiles(fs, conf, htd, family, cacheConfig,
7171
EnvironmentEdgeManager.currentTime());
7272
}
7373

@@ -107,7 +107,7 @@ public int run(String[] args) throws Exception {
107107
throw new IOException(
108108
"The minVersions of the column family is not 0, could not be handled by this cleaner");
109109
}
110-
cleanExpiredMobFiles(tableName, family);
110+
cleanExpiredMobFiles(htd, family);
111111
return 0;
112112
} finally {
113113
try {

‎hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.hadoop.hbase.regionserver.BloomType;
3030
import org.apache.hadoop.hbase.regionserver.HStoreFile;
3131
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
32+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
3233
import org.apache.yetus.audience.InterfaceAudience;
3334

3435
/**
@@ -133,11 +134,11 @@ public void close() throws IOException {
133134
* @param cacheConf The CacheConfig.
134135
* @return An instance of the MobFile.
135136
*/
136-
public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf)
137-
throws IOException {
137+
public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf,
138+
StoreFileTracker sft) throws IOException {
138139
// XXX: primaryReplica is only used for constructing the key of block cache so it is not a
139140
// critical problem if we pass the wrong value, so here we always pass true. Need to fix later.
140-
HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
141+
HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft);
141142
return new MobFile(sf);
142143
}
143144
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333
import org.apache.hadoop.fs.FileSystem;
3434
import org.apache.hadoop.fs.Path;
3535
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36+
import org.apache.hadoop.hbase.regionserver.StoreContext;
37+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
38+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
3639
import org.apache.hadoop.hbase.util.IdLock;
3740
import org.apache.yetus.audience.InterfaceAudience;
3841
import org.slf4j.Logger;
@@ -198,9 +201,11 @@ public void evictFile(String fileName) {
198201
* @param cacheConf The current MobCacheConfig
199202
* @return A opened mob file.
200203
*/
201-
public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException {
204+
public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf,
205+
StoreContext storeContext) throws IOException {
206+
StoreFileTracker sft = StoreFileTrackerFactory.create(conf, false, storeContext);
202207
if (!isCacheEnabled) {
203-
MobFile mobFile = MobFile.create(fs, path, conf, cacheConf);
208+
MobFile mobFile = MobFile.create(fs, path, conf, cacheConf, sft);
204209
mobFile.open();
205210
return mobFile;
206211
} else {
@@ -214,7 +219,7 @@ public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws
214219
if (map.size() > mobFileMaxCacheSize) {
215220
evict();
216221
}
217-
cached = CachedMobFile.create(fs, path, conf, cacheConf);
222+
cached = CachedMobFile.create(fs, path, conf, cacheConf, sft);
218223
cached.open();
219224
map.put(fileName, cached);
220225
miss.increment();

0 commit comments

Comments
 (0)