//HRegionServer类的splitRegion方法,该方法封装了手动split的实现逻辑
public void splitRegion(HRegionInfo regionInfo, byte[] splitPoint)
throws NotServingRegionException, IOException {
region.flushcache(); //刷新memstore
region.forceSplit(splitPoint);//设置splitRequest状态为true,以便强制split
//通过线程池完成split,线程池大小通过hbase.regionserver.thread.split配置(默认为1),具体的split逻辑请参阅接下来的SplitRequest的run方法
compactSplitThread.requestSplit(region, region.checkSplit());
}
public byte[] checkSplit() {
// Can't split ROOT/META
if (this.regionInfo.isMetaTable()) {
return null;
}
//如果splitRequest=true或者发现一个大小达到split条件的store(注意:一个store对应一个列族,有多个storefile文件和一个Memstore)则返回true,如果不是强制split(splitRequest=false)且该region下存在一个storefile是reference类型的文件则不能split(reference文件只会存在split期间)
if (!splitPolicy.shouldSplit()) {
return null;
}
//如果在split时指定了rowkey直接返回,否则执行以下逻辑:循环所有的store,找出最大并且不包含reference的store,采用该store的splitPoint作为region的splitPoint,store的splitPoint采用最大的storeFile,的中间rowKey作为splitPoint,因此split实际上并不是完全的等分,可能一个region会比另一个大很多
byte[] ret = splitPolicy.getSplitPoint();
if (ret != null) {
try {
checkRow(ret, "calculated split");
} catch (IOException e) {
LOG.error("Ignoring invalid split", e);
return null;
}
}
return ret;
}
//SplitRequest的run方法
public void run() {
SplitTransaction st = new SplitTransaction(parent, midKey);
if (!st.prepare()) return;
st.execute(this.server, this.server);
}
// SplitTransaction的prepare方法
public boolean prepare() {
//如果region被close或者正在close或者有reference类型的storefile则不split,只有split过程中才会产生reference的storefile文件
if (!this.parent.isSplittable()) return false;
HRegionInfo hri = this.parent.getRegionInfo();
byte [] startKey = hri.getStartKey();
byte [] endKey = hri.getEndKey();
long rid = getDaughterRegionIdTimestamp(hri);
//创建两个新的region对象
this.hri_a = new HRegionInfo(hri.getTableName(), startKey, this.splitrow, false, rid);
this.hri_b = new HRegionInfo(hri.getTableName(), this.splitrow, endKey, false, rid);
return true;
}
// SplitTransaction的excuete方法
public PairOfSameType<HRegion> execute(final Server server,
final RegionServerServices services)
throws IOException {
//关闭被split的region,生成两个新的region:A B
PairOfSameType<HRegion> regions = createDaughters(server, services);
//打开split后生成的A B两个region
openDaughters(server, services, regions.getFirst(), regions.getSecond());
//修改zk中保存的split状态由spliting为split,master获知该状态后会删除zk中的该状态节点
该方法会在循环中检测master是否获得了通知,每次检测会休眠100ms
transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
return regions;
}
PairOfSameType<HRegion> createDaughters(final Server server,
final RegionServerServices services) throws IOException {
//设置split超时时间(默认为3000ms,可通过hbase.regionserver.fileSplitTimeout项配置)
boolean testing = server == null? true:
server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
this.fileSplitTimeout = testing ? this.fileSplitTimeout :
server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout", this.fileSplitTimeout);
//在zk创建一个临时的节点,保存split状态:
RS_ZK_REGION_CLOSED (2), // RS has finished closing a region
RS_ZK_REGION_OPENING (3), // RS is in process of opening a region
RS_ZK_REGION_OPENED (4), // RS has finished opening a region
RS_ZK_REGION_SPLITTING (5), // RS has started a region split
RS_ZK_REGION_SPLIT (6), // RS split has completed.
RS_ZK_REGION_FAILED_OPEN (7), // RS failed to open a region
if (server != null && server.getZooKeeper() != null) {
try {
createNodeSplitting(server.getZooKeeper(), this.parent.getRegionInfo(), server.getServerName());
} catch (KeeperException e) {
throw new IOException("Failed creating SPLITTING znode on " +this.parent.getRegionNameAsString(), e);
}
}
this.journal.add(JournalEntry.SET_SPLITTING_IN_ZK);
if (server != null && server.getZooKeeper() != null) {
try {
this.znodeVersion = transitionNodeSplitting(server.getZooKeeper(),
this.parent.getRegionInfo(), server.getServerName(), -1);
} catch (KeeperException e) {
throw new IOException("Failed setting SPLITTING znode on " + this.parent.getRegionNameAsString(), e);
}
}
//在hdfs创建一个split目录:{ region dir}/splits
createSplitDir(this.parent.getFilesystem(), this.splitdir);
this.journal.add(JournalEntry.CREATE_SPLIT_DIR);
List<StoreFile> hstoreFilesToSplit = null;
Exception exceptionToThrow = null;
try{
//关闭当前region,不再提供任何服务,在关闭region前会等待该region的所有compact和flush均完成(compact和flush是异步的操作),并且如果发现memstore的大小等于或者大于hbase.hregion.preclose.flush.size配置的值(默认为5M)时,会进行preFlush(close前的flush),这些操作完成后才设置region的closing状态为true,设置closing状态后会分配一个writelock,此时拒绝任何服务了,直到关闭完全,设置closed为true,最后释放写锁
hstoreFilesToSplit = this.parent.close(false);
} catch (Exception e) {
exceptionToThrow = e;
}
//从RS中onlineRegions中删除被split的region,在下线region后的这段时间里client请求该region的数据会失败,client会不断尝试(尝试时间间隔会随着次数的增加而增大,前几次为采用hbase.client.pause配置的值,默认值为1000ms)直到split后形成的A B region上线,此过程比较快,没有复杂的处理,总之split时可能会出现超过1秒的访问。
if (!testing) {
services.removeFromOnlineRegions(this.parent.getRegionInfo().getEncodedName());
}
this.journal.add(JournalEntry.OFFLINED_PARENT);
//通过线程池split storeFile,池大小=storeFile个数,具体的split逻辑请参阅StoreFileSplitter的splitStoreFile()
splitStoreFiles(this.splitdir, hstoreFilesToSplit);
this.journal.add(JournalEntry.STARTED_REGION_A_CREATION);
HRegion a = createDaughterRegion(this.hri_a, this.parent.rsServices);
this.journal.add(JournalEntry.STARTED_REGION_B_CREATION);
HRegion b = createDaughterRegion(this.hri_b, this.parent.rsServices);
this.journal.add(JournalEntry.PONR);
if (!testing) {
//在.META.表中下线split的region,修改.META.表的该region信息,把offline split设置为true,并且添加两列:splitA和splitB
MetaEditor.offlineParentInMeta(server.getCatalogTracker(),
this.parent.getRegionInfo(), a.getRegionInfo(), b.getRegionInfo());
}
return new PairOfSameType<HRegion>(a, b);
}
//StoreFileSplitter类的splitStoreFile逻辑
private void splitStoreFile(final StoreFile sf, final Path splitdir)
throws IOException {
FileSystem fs = this.parent.getFilesystem();
byte [] family = sf.getFamily();
String encoded = this.hri_a.getEncodedName();
Path storedir = Store.getStoreHomedir(splitdir, encoded, family);
StoreFile.split(fs, storedir, sf, this.splitrow, Range.bottom);
encoded = this.hri_b.getEncodedName();
storedir = Store.getStoreHomedir(splitdir, encoded, family);
StoreFile.split(fs, storedir, sf, this.splitrow, Range.top);
}
// StoreFile的split方法
static Path split(final FileSystem fs,
final Path splitDir,
final StoreFile f,
final byte [] splitRow,
final Reference.Range range)
throws IOException {
// 检查split的row是否是在该store file范围内
if (range == Reference.Range.bottom) {
KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
byte[] firstKey = f.createReader().getFirstKey();
if (f.getReader().getComparator().compare(splitKey.getBuffer(),
splitKey.getKeyOffset(), splitKey.getKeyLength(),
firstKey, 0, firstKey.length) < 0) {
return null;
}
}
else {
KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
byte[] lastKey = f.createReader().getLastKey();
if (f.getReader().getComparator().compare(splitKey.getBuffer(),
splitKey.getKeyOffset(), splitKey.getKeyLength(),
lastKey, 0, lastKey.length) > 0) {
return null;
}
}
//写reference store,生成的reference stroe的名称格式例如:0959f79e6f7141aba1d81450ac891884.a00556374994fa5b3369e884b45492d3,其中a00556374994fa5b3369e884b45492d3为被split的region的id,0959f79e6f7141aba1d81450ac891884为引用的storeFile的名称,该引用的storefile的内容是用于split的中间rowkey,两个新的region的reference文件的个数均与split region的storeFile文件个数相同,通过hbase的管理界面看到
Reference r = new Reference(splitRow, range);
String parentRegionName = f.getPath().getParent().getParent().getName();
Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
return r.write(fs, p);
}
//打开split生成的两个region
void openDaughters(final Server server,
final RegionServerServices services, HRegion a, HRegion b)
throws IOException {
DaughterOpener aOpener = new DaughterOpener(server, a);
DaughterOpener bOpener = new DaughterOpener(server, b);
aOpener.start();
bOpener.start();
aOpener.join();//等待open完成
bOpener.join();
//如果包含reference的storefile或者storefile数量超过了配置的限制大小时会requestCompact,compact后,reference文件变为实际文件
//增加region信息到.META.表
services.postOpenDeployTasks(b, server.getCatalogTracker(), true);
//添加region对象到RegionServer的onlineRegions列表中,此时可以对外服务
services.addToOnlineRegions(b);
services.postOpenDeployTasks(a, server.getCatalogTracker(), true);
services.addToOnlineRegions(a);
}
}
相关推荐
HashMap中红黑树TreeNode的split方法源码解读,对split方法源码的上下文/变量定义,及所调用的关键方法都给出了详细解释说明,欢迎指正
详细介绍C#中Split的用法,各种情况下用法及举例介绍
split_dos
本文档详细介绍了如何在Oracle数据库中实现像C#等其它编程语言中Split函数来拆分字符的功能。例子简单易懂,并附有详细的实现过程。
非常实用的VB.NET中使用split的方法,介绍详细
C# split 函数的用法,比较详细,具体的解读该函数的使用方法。
JAVA_split_用法,split详解,记得好评哦
java中String类的intern、split方法的详细讲解。
IIS补丁,解决WIN7下SPLIT函数出错
PDF Split pdf 分割工具 非常简洁
Sql in split Sql in split Sql in split Sql in split
C#中Split用法,帮助大家学习交流哈
文件分割工具 Super Split 使用说明 一、特点 本软件最大特点是能够自动检测A盘剩余容量,自适应A盘。 二、使用方法: (1)split -xxxK filename.ext 其中“K”不能省略,“K”表示以KB为单位进行...
THIS ALGORITHM USED TO SPLIT THE IMAGE INTO DIFFERENT REGION.
SplitButton 分裂按钮 WPF中制作分裂按钮的DEMO,使用很方便,制作华丽的SplitButton 分裂按钮
Java中Split的用法及注意, 本文只是大概说了一下java中split的用法,其他方面要你们去查看了。
SplitContainer带箭头收缩美化,希望使用时尽量保留原作者注释,尊重原著。
Free software for agile handling of Trados Studio tasks.
oracle ebs Split An Order Line.